diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml new file mode 100644 index 000000000..4a55d650c --- /dev/null +++ b/.github/workflows/deploy-prod.yml @@ -0,0 +1,152 @@ +name: Deploy to Prod environments. +on: + workflow_dispatch: + inputs: + network: + type: choice + options: + - mainnet + - testnet + description: mainnet or testnet network + required: true + version: + description: What mainnet version number is this deployment? (e.g. v0.1.0) + required: true + + +env: + PROJECT_PROD: "pagoda-discovery-platform-prod" + REGION: "us-east1" + IMAGE: us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery:${{ github.sha }} + +jobs: + build-mpc-recovery: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + name: "Checkout mpc-recovery" + + - name: Login to Artifact Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGION }}-docker.pkg.dev + username: _json_key + password: ${{ secrets.GCP_CREDENTIALS_DEV }} + + - name: Build Docker image and push to Google Artifact Registry + id: docker-push-tagged + uses: docker/build-push-action@v4 + with: + push: true + file: ./Dockerfile + tags: "${{ env.IMAGE }}:${{ github.sha }}" + + deploy-mpc-recovery-testnet: + runs-on: ubuntu-latest + if: github.event.inputs.network == 'testnet' + needs: build-mpc-recovery + env: + name: PROD + steps: + - uses: actions/checkout@v3 + name: "Checkout mpc-recovery" + + - name: "Authenticate to GCloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GCP_CREDENTIALS_DEV }}" + + - name: Deploy leader to Cloud Run Testnet + id: deploy-leader + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-leader-testnet + region: us-east1 + project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" + + - name: Deploy signer to Cloud Run testnet + id: deploy-signer-0-testnet + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-0-testnet + region: us-east1 + project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" + + - name: Deploy signer to Cloud Run testnet + id: deploy-signer-1-testnet + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-1-testnet + region: us-east1 + project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" + + - name: Deploy signer to Cloud Run testnet + id: deploy-signer-2-testnet + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-2-testnet + region: us-east1 + project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" + + deploy-mpc-recovery-mainnet: + runs-on: ubuntu-latest + if: github.event.inputs.network == 'mainnet' + needs: build-mpc-recovery + env: + name: PROD + steps: + - uses: actions/checkout@v3 + name: "Checkout mpc-recovery" + + - name: "Authenticate to GCloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GCP_CREDENTIALS_DEV }}" + + - name: Deploy leader to Cloud Run mainnet + id: deploy-leader + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-leader-mainnet + region: us-east1 + project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" + + - name: Deploy signer to Cloud Run mainnet + id: deploy-signer-0 + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-0-mainnet + region: us-east1 + project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" + + - name: Deploy signer to Cloud Run mainnet + id: deploy-signer-1 + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-1-mainnet + region: us-east1 + project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" + + - name: Deploy signer to Cloud Run mainnet + id: deploy-signer-2 + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-2-mainnet + region: us-east1 + project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" \ No newline at end of file diff --git a/.github/workflows/terraform-dev.yml b/.github/workflows/terraform-dev.yml index 5fdc9c98d..4bef552b9 100644 --- a/.github/workflows/terraform-dev.yml +++ b/.github/workflows/terraform-dev.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write defaults: run: - working-directory: ./infra + working-directory: ./infra/mpc-recovery-dev steps: - name: Checkout uses: actions/checkout@v3 @@ -54,6 +54,11 @@ jobs: env: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} + # Build Docker image. + - name: Docker Image + id: build + run: docker build .. -t near/mpc-recovery + # Generates an execution plan for Terraform - name: Terraform Plan id: plan diff --git a/.github/workflows/terraform-feature-env-destroy.yml b/.github/workflows/terraform-feature-env-destroy.yml index d59626cae..65723ad16 100644 --- a/.github/workflows/terraform-feature-env-destroy.yml +++ b/.github/workflows/terraform-feature-env-destroy.yml @@ -14,7 +14,7 @@ jobs: pull-requests: write defaults: run: - working-directory: ./infra + working-directory: ./infra/mpc-recovery-dev env: PR_NUMBER: ${{ github.event.number }} steps: diff --git a/.github/workflows/terraform-feature-env.yml b/.github/workflows/terraform-feature-env.yml index 6b1d4ba5e..35c8c7efa 100644 --- a/.github/workflows/terraform-feature-env.yml +++ b/.github/workflows/terraform-feature-env.yml @@ -15,7 +15,7 @@ jobs: checks: read defaults: run: - working-directory: ./infra + working-directory: ./infra/mpc-recovery-dev env: PR_NUMBER: ${{ github.event.number }} steps: diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 000000000..8051df6ed --- /dev/null +++ b/infra/README.md @@ -0,0 +1,20 @@ +# MPC Recovery Infrastructure Overview + +There are currently 3 mostly static environments for MPC + - Mainnet (production) + - Testnet (production) + - Dev (development) + + ## Mainnet/Testnet + + Mainnet and Testnet infra code is in the directory `mpc-recovery-prod` and is built off of the `main` GitHub Branch + - This environment should be deployed via the GHA pipeline `deploy-prod.yml` manually in order to prevent unwanted changes + - Both Mainnet and Testnet are treated as production environments + + ## Dev + + The Dev environment infra code is located in the `mpc-recovery-dev` directory and is built off of the `develop` GitHub Branch + - This should be used as the main development environment + - Every time a pull request is opened up against the `develop` branch, a new, ephemeral environment is created with your changes + - *Note: These environments will have the associated PR number appended to all resources* + - When a pull request is approved and merged into the `develop` branch, a new revision is deployed to the static Dev environment with the PRs changes and the PRs ephemeral environment is destroyed \ No newline at end of file diff --git a/infra/modules/internal_cloudrun_lb/main.tf b/infra/modules/internal_cloudrun_lb/main.tf new file mode 100644 index 000000000..f6492bf7a --- /dev/null +++ b/infra/modules/internal_cloudrun_lb/main.tf @@ -0,0 +1,50 @@ +resource "google_compute_region_network_endpoint_group" "default_neg" { + name = "${var.name}-neg" + project = var.project_id + network_endpoint_type = "SERVERLESS" + region = var.region + cloud_run { + service = var.service_name + } +} + +resource "google_compute_region_backend_service" "default" { + name = "${var.name}-backend-service" + project = var.project_id + region = var.region + protocol = "HTTP" + load_balancing_scheme = "INTERNAL_MANAGED" + timeout_sec = 30 + backend { + group = google_compute_region_network_endpoint_group.default_neg.id + balancing_mode = "UTILIZATION" + capacity_scaler = 1.0 + } +} + +resource "google_compute_region_url_map" "default" { + name = "${var.name}-url-map" + project = var.project_id + region = var.region + default_service = google_compute_region_backend_service.default.id +} + +resource "google_compute_region_target_http_proxy" "default" { + name = "${var.name}-http-proxy" + region = var.region + project = var.project_id + url_map = google_compute_region_url_map.default.id +} + +resource "google_compute_forwarding_rule" "default" { + name = "${var.name}-forwarding-rule" + project = var.project_id + region = var.region + ip_protocol = "TCP" + load_balancing_scheme = "INTERNAL_MANAGED" + port_range = "80" + target = google_compute_region_target_http_proxy.default.id + network = var.network_id + subnetwork = var.subnetwork_id + network_tier = "PREMIUM" +} diff --git a/infra/modules/internal_cloudrun_lb/variables.tf b/infra/modules/internal_cloudrun_lb/variables.tf new file mode 100644 index 000000000..c09e1d886 --- /dev/null +++ b/infra/modules/internal_cloudrun_lb/variables.tf @@ -0,0 +1,29 @@ +variable "name" { + type = string + description = "The name to use as prefix for load balancer resources." +} + +variable "service_name" { + type = string + description = "The cloud run service name" +} + +variable "project_id" { + type = string + description = "The GCP project these resources belong to" +} + +variable "region" { + type = string + description = "The region where resources will live." +} + +variable "network_id" { + type = string + description = "The VPC network to connect to." +} + +variable "subnetwork_id" { + type = string + description = "Subnet for hosting the load balancer." +} \ No newline at end of file diff --git a/infra/modules/leader/main.tf b/infra/modules/leader/main.tf index 608f6c606..06928252d 100644 --- a/infra/modules/leader/main.tf +++ b/infra/modules/leader/main.tf @@ -1,11 +1,18 @@ resource "google_cloud_run_v2_service" "leader" { - name = "mpc-recovery-leader-${var.env}" + name = var.service_name location = var.region ingress = "INGRESS_TRAFFIC_ALL" template { service_account = var.service_account_email + annotations = var.metadata_annotations == null ? null : var.metadata_annotations + + vpc_access { + connector = var.connector_id + egress = "PRIVATE_RANGES_ONLY" + } + scaling { min_instance_count = 1 max_instance_count = 1 @@ -48,7 +55,7 @@ resource "google_cloud_run_v2_service" "leader" { value_source { secret_key_ref { secret = var.account_creator_sk_secret_id - version = "1" + version = "latest" } } } @@ -82,7 +89,6 @@ resource "google_cloud_run_v2_service" "leader" { ports { container_port = 3000 } - resources { cpu_idle = false diff --git a/infra/modules/leader/variables.tf b/infra/modules/leader/variables.tf index 6e0655e58..fe51e5b82 100644 --- a/infra/modules/leader/variables.tf +++ b/infra/modules/leader/variables.tf @@ -6,6 +6,16 @@ variable "project" { type = string } +variable "connector_id" { + description = "VPC connector ID for internal traffic" +} + +variable "metadata_annotations" { + type = map(any) + default = null + description = "Annotations for the metadata associated with this Service." +} + variable "region" { type = string } @@ -48,6 +58,9 @@ variable "fast_auth_partners_secret_id" { type = string } +variable "service_name" { + type = string +} variable "jwt_signature_pk_url" { type = string } diff --git a/infra/modules/signer/main.tf b/infra/modules/signer/main.tf index e22c36158..7800b806d 100644 --- a/infra/modules/signer/main.tf +++ b/infra/modules/signer/main.tf @@ -1,11 +1,18 @@ resource "google_cloud_run_v2_service" "signer" { - name = "mpc-recovery-signer-${var.node_id}-${var.env}" + name = var.service_name location = var.region ingress = "INGRESS_TRAFFIC_ALL" template { service_account = var.service_account_email + annotations = var.metadata_annotations == null ? null : var.metadata_annotations + + vpc_access { + connector = var.connector_id == null ? null : var.connector_id + egress = "PRIVATE_RANGES_ONLY" + } + scaling { min_instance_count = 1 max_instance_count = 1 @@ -62,6 +69,8 @@ resource "google_cloud_run_v2_service" "signer" { container_port = 3000 } + + resources { cpu_idle = false diff --git a/infra/modules/signer/variables.tf b/infra/modules/signer/variables.tf index 5c7e2670d..4f68e07b0 100644 --- a/infra/modules/signer/variables.tf +++ b/infra/modules/signer/variables.tf @@ -16,6 +16,16 @@ variable "service_account_email" { variable "docker_image" { } +variable "connector_id" { + description = "VPC connector ID for internal traffic" +} + +variable "metadata_annotations" { + type = map(any) + default = null + description = "Annotations for the metadata associated with this Service." +} + # Application variables variable "node_id" { } @@ -29,6 +39,9 @@ variable "sk_share_secret_id" { type = string } -variable "jwt_signature_pk_url" { +variable "service_name" { type = string } + +variable "jwt_signature_pk_url" { +} diff --git a/infra/backend-config-dev.tfvars b/infra/mpc-recovery-dev/backend-config-dev.tfvars similarity index 100% rename from infra/backend-config-dev.tfvars rename to infra/mpc-recovery-dev/backend-config-dev.tfvars diff --git a/infra/main.tf b/infra/mpc-recovery-dev/main.tf similarity index 77% rename from infra/main.tf rename to infra/mpc-recovery-dev/main.tf index 6f32cce90..c6865e82b 100644 --- a/infra/main.tf +++ b/infra/mpc-recovery-dev/main.tf @@ -1,5 +1,6 @@ terraform { backend "gcs" { + bucket = "mpc-recovery-terraform-dev" prefix = "state/mpc-recovery" } @@ -33,11 +34,12 @@ locals { } data "external" "git_checkout" { - program = ["${path.module}/scripts/get_sha.sh"] + program = ["${path.module}/../scripts/get_sha.sh"] } provider "google" { credentials = local.credentials + # credentials = file("~/.config/gcloud/application_default_credentials.json") project = var.project region = var.region @@ -58,6 +60,7 @@ resource "google_service_account_iam_binding" "serivce-account-iam" { members = [ "serviceAccount:${local.client_email}", + # "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" ] } @@ -98,19 +101,42 @@ resource "google_secret_manager_secret_iam_member" "fast_auth_partners_secret_ac member = "serviceAccount:${google_service_account.service_account.email}" } +module "mpc-signer-lb" { + + count = length(var.signer_configs) + source = "../modules/internal_cloudrun_lb" + name = "mpc-${var.env}-signer-${count.index}" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id + project_id = var.project + region = "us-central1" + service_name = "mpc-recovery-signer-${count.index}-${var.env}" +} + +module "mpc-leader-lb" { + source = "../modules/internal_cloudrun_lb" + name = "mpc-${var.env}-leader" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id + project_id = var.project + region = "us-central1" + service_name = "mpc-recovery-leader-${var.env}" +} /* * Create multiple signer nodes */ module "signer" { count = length(var.signer_configs) - source = "./modules/signer" + source = "../modules/signer" env = var.env + service_name = "mpc-recovery-signer-${count.index}-${var.env}" project = var.project region = var.region zone = var.zone service_account_email = google_service_account.service_account.email docker_image = var.docker_image + connector_id = var.dev-connector node_id = count.index @@ -129,14 +155,16 @@ module "signer" { * Create leader node */ module "leader" { - source = "./modules/leader" + source = "../modules/leader" env = var.env + service_name = "mpc-recovery-leader-${var.env}" project = var.project region = var.region zone = var.zone service_account_email = google_service_account.service_account.email docker_image = var.docker_image + connector_id = var.prod-connector signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) near_rpc = local.workspace.near_rpc diff --git a/infra/migration.py b/infra/mpc-recovery-dev/migration.py similarity index 100% rename from infra/migration.py rename to infra/mpc-recovery-dev/migration.py diff --git a/infra/output.tf b/infra/mpc-recovery-dev/output.tf similarity index 100% rename from infra/output.tf rename to infra/mpc-recovery-dev/output.tf diff --git a/infra/terraform-dev.tfvars b/infra/mpc-recovery-dev/terraform-dev.tfvars similarity index 91% rename from infra/terraform-dev.tfvars rename to infra/mpc-recovery-dev/terraform-dev.tfvars index 0ab14fe3e..8568423aa 100644 --- a/infra/terraform-dev.tfvars +++ b/infra/mpc-recovery-dev/terraform-dev.tfvars @@ -1,6 +1,6 @@ env = "dev" project = "pagoda-discovery-platform-dev" -docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery" +docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:f405e5594e666ef6a6e5ccf701c0d70c9673d4fe" account_creator_id = "mpc-recovery-dev-creator.testnet" account_creator_sk_secret_id = "mpc-recovery-account-creator-sk-dev" diff --git a/infra/variables.tf b/infra/mpc-recovery-dev/variables.tf similarity index 50% rename from infra/variables.tf rename to infra/mpc-recovery-dev/variables.tf index 5917e1d50..2719c4d57 100644 --- a/infra/variables.tf +++ b/infra/mpc-recovery-dev/variables.tf @@ -50,16 +50,45 @@ variable "signer_configs" { })) } + +variable "dev-connector" { + default = "projects/pagoda-shared-infrastructure/locations/us-east1/connectors/dev-connector1" +} + +variable "prod-connector" { + default = "projects/pagoda-shared-infrastructure/locations/us-east1/connectors/prod-us-east1-connector" +} + +data "google_compute_subnetwork" "dev_subnetwork" { + name = "dev-us-central1" + project = "pagoda-shared-infrastructure" + region = "us-central1" +} + +data "google_compute_subnetwork" "prod_subnetwork" { + name = "prod-us-central1" + project = "pagoda-shared-infrastructure" + region = "us-central1" +} + +data "google_compute_network" "dev_network" { + name = "dev" + project = "pagoda-shared-infrastructure" +} + +data "google_compute_network" "prod_network" { + name = "prod" + project = "pagoda-shared-infrastructure" +} + variable "jwt_signature_pk_url" { type = string } variable "otlp_endpoint" { - type = string - default = "http://localhost:4317" + type = string } variable "opentelemetry_level" { - type = string - default = "off" -} + type = string +} \ No newline at end of file diff --git a/infra/backend-config-prod.tfvars b/infra/mpc-recovery-prod/backend-config-prod.tfvars similarity index 100% rename from infra/backend-config-prod.tfvars rename to infra/mpc-recovery-prod/backend-config-prod.tfvars diff --git a/infra/mpc-recovery-prod/main.tf b/infra/mpc-recovery-prod/main.tf new file mode 100644 index 000000000..a43b67d78 --- /dev/null +++ b/infra/mpc-recovery-prod/main.tf @@ -0,0 +1,270 @@ +terraform { + backend "gcs" { + bucket = "mpc-recovery-terraform-prod" + prefix = "state/mpc-recovery" + } + + required_providers { + google = { + source = "hashicorp/google" + version = "4.73.0" + } + } +} + +locals { + credentials = var.credentials != null ? var.credentials : file(var.credentials_file) + client_email = jsondecode(local.credentials).client_email + client_id = jsondecode(local.credentials).client_id + + env = { + defaults = { + near_rpc = "https://rpc.testnet.near.org" + near_root_account = "testnet" + } + testnet = { + } + mainnet = { + near_rpc = "https://rpc.mainnet.near.org" + near_root_account = "near" + } + } + + workspace = merge(local.env["defaults"], contains(keys(local.env), terraform.workspace) ? local.env[terraform.workspace] : local.env["defaults"]) +} + +data "external" "git_checkout" { + program = ["${path.module}/../scripts/get_sha.sh"] +} + +provider "google" { + credentials = local.credentials + # credentials = file("~/.config/gcloud/application_default_credentials.json") + + project = var.project + region = var.region + zone = var.zone +} + +/* + * Create brand new service account with basic IAM + */ +resource "google_service_account" "service_account" { + account_id = "mpc-recovery-prod" + display_name = "MPC Recovery prod Account" +} + +resource "google_service_account_iam_binding" "serivce-account-iam" { + service_account_id = google_service_account.service_account.name + role = "roles/iam.serviceAccountUser" + + members = [ + "serviceAccount:${local.client_email}", + # "serviceAccount:mpc-recovery@pagoda-discovery-platform-prod.iam.gserviceaccount.com" + ] +} + +resource "google_project_iam_member" "service-account-datastore-user" { + project = var.project + role = "roles/datastore.user" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +/* + * Ensure service account has access to Secret Manager variables + */ +resource "google_secret_manager_secret_iam_member" "cipher_key_secret_access" { + count = length(var.signer_configs) + + secret_id = var.signer_configs[count.index].cipher_key_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "secret_share_secret_access" { + count = length(var.signer_configs) + + secret_id = var.signer_configs[count.index].sk_share_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "oidc_providers_secret_access" { + secret_id = var.oidc_providers_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "account_creator_secret_access" { + secret_id = var.account_creator_sk_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "fast_auth_partners_secret_access" { + secret_id = var.fast_auth_partners_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +module "mpc-signer-lb-mainnet" { + + count = length(var.signer_configs) + source = "../modules/internal_cloudrun_lb" + name = "mpc-prod-signer-${count.index}-mainnet" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id + project_id = var.project + region = "us-central1" + service_name = "mpc-recovery-signer-${count.index}-mainnet" +} + +module "mpc-signer-lb-testnet" { + + count = length(var.signer_configs) + source = "../modules/internal_cloudrun_lb" + name = "mpc-prod-signer-${count.index}-testnet" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id + project_id = var.project + region = "us-central1" + service_name = "mpc-recovery-signer-${count.index}-testnet" +} + +module "mpc-leader-lb-mainnet" { + source = "../modules/internal_cloudrun_lb" + name = "mpc-prod-leader-mainnet" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id + project_id = var.project + region = "us-central1" + service_name = "mpc-recovery-leader-mainnet" +} + +module "mpc-leader-lb-testnet" { + source = "../modules/internal_cloudrun_lb" + name = "mpc-prod-leader-testnet" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id + project_id = var.project + region = "us-central1" + service_name = "mpc-recovery-leader-testnet" +} +/* + * Create multiple signer nodes + */ +module "signer-mainnet" { + count = length(var.signer_configs) + source = "../modules/signer" + + env = "prod" + service_name = "mpc-recovery-signer-${count.index}-mainnet" + project = var.project + region = var.region + zone = var.zone + service_account_email = google_service_account.service_account.email + docker_image = var.docker_image + connector_id = var.prod-connector + jwt_signature_pk_url = var.jwt_signature_pk_url + + node_id = count.index + + cipher_key_secret_id = var.signer_configs[count.index].cipher_key_secret_id + sk_share_secret_id = var.signer_configs[count.index].sk_share_secret_id + + depends_on = [ + google_secret_manager_secret_iam_member.cipher_key_secret_access, + google_secret_manager_secret_iam_member.secret_share_secret_access, + google_secret_manager_secret_iam_member.oidc_providers_secret_access + ] +} + +module "signer-testnet" { + count = length(var.signer_configs) + source = "../modules/signer" + + env = "prod" + service_name = "mpc-recovery-signer-${count.index}-testnet" + project = var.project + region = var.region + zone = var.zone + service_account_email = google_service_account.service_account.email + docker_image = var.docker_image + connector_id = var.prod-connector + jwt_signature_pk_url = var.jwt_signature_pk_url + + node_id = count.index + + cipher_key_secret_id = var.signer_configs[count.index].cipher_key_secret_id + sk_share_secret_id = var.signer_configs[count.index].sk_share_secret_id + + depends_on = [ + google_secret_manager_secret_iam_member.cipher_key_secret_access, + google_secret_manager_secret_iam_member.secret_share_secret_access, + google_secret_manager_secret_iam_member.oidc_providers_secret_access + ] +} + +/* + * Create leader node + */ +module "leader-mainnet" { + source = "../modules/leader" + + env = "prod" + service_name = "mpc-recovery-leader-mainnet" + project = var.project + region = var.region + zone = var.zone + service_account_email = google_service_account.service_account.email + docker_image = var.docker_image + connector_id = var.prod-connector + jwt_signature_pk_url = var.jwt_signature_pk_url + opentelemetry_level = var.opentelemetry_level + otlp_endpoint = var.otlp_endpoint + + signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) + near_rpc = local.workspace.near_rpc + near_root_account = local.workspace.near_root_account + account_creator_id = var.account_creator_id + + account_creator_sk_secret_id = var.account_creator_sk_secret_id + fast_auth_partners_secret_id = var.fast_auth_partners_secret_id + + depends_on = [ + google_secret_manager_secret_iam_member.account_creator_secret_access, + google_secret_manager_secret_iam_member.fast_auth_partners_secret_access, + module.signer + ] +} + +module "leader-testnet" { + source = "../modules/leader" + + env = "prod" + service_name = "mpc-recovery-leader-testnet" + project = var.project + region = var.region + zone = var.zone + service_account_email = google_service_account.service_account.email + docker_image = var.docker_image + connector_id = var.prod-connector + jwt_signature_pk_url = var.jwt_signature_pk_url + opentelemetry_level = var.opentelemetry_level + otlp_endpoint = var.otlp_endpoint + + signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) + near_rpc = local.workspace.near_rpc + near_root_account = local.workspace.near_root_account + account_creator_id = var.account_creator_id + + + account_creator_sk_secret_id = var.account_creator_sk_secret_id + fast_auth_partners_secret_id = var.fast_auth_partners_secret_id + + depends_on = [ + google_secret_manager_secret_iam_member.account_creator_secret_access, + google_secret_manager_secret_iam_member.fast_auth_partners_secret_access, + module.signer + ] +} diff --git a/infra/mpc-recovery-prod/migration.py b/infra/mpc-recovery-prod/migration.py new file mode 100644 index 000000000..afbdc3d2b --- /dev/null +++ b/infra/mpc-recovery-prod/migration.py @@ -0,0 +1,22 @@ +from google.oauth2 import service_account +from google.cloud import datastore + +credentials_source = service_account.Credentials.from_service_account_file( + '../source-service-keys.json') +client_source = datastore.Client(project="pagoda-discovery-platform-dev", credentials=credentials_source) + +credentials_target = service_account.Credentials.from_service_account_file( + '../target-service-keys.json') +client_target = datastore.Client(project="pagoda-discovery-platform-prod", credentials=credentials_target) + +print('Fetching source entities') +query = credentials_source.query(kind="EncryptedUserCredentials-dev") +entities = [] +for entity in list(query.fetch()): + entity.key = client_target.key('EncryptedUserCredentials-mainnet').completed_key(entity.key.id_or_name) + print(entity.key) + print(entity) + entities.append(entity) + +print("Uploading a total of " + str(len(entities)) + " entities to target") +client_target.put_multi(entities) \ No newline at end of file diff --git a/infra/mpc-recovery-prod/output.tf b/infra/mpc-recovery-prod/output.tf new file mode 100644 index 000000000..3447956d7 --- /dev/null +++ b/infra/mpc-recovery-prod/output.tf @@ -0,0 +1,3 @@ +output "leader_node" { + value = module.leader.node.uri +} diff --git a/infra/mpc-recovery-prod/variables.tf b/infra/mpc-recovery-prod/variables.tf new file mode 100644 index 000000000..f48a2021f --- /dev/null +++ b/infra/mpc-recovery-prod/variables.tf @@ -0,0 +1,97 @@ +variable "env" { +} + +variable "project" { +} + +variable "credentials_file" { + default = null +} + +variable "credentials" { + default = null +} + +variable "region" { + default = "us-east1" +} + +variable "zone" { + default = "us-east1-c" +} + +variable "docker_image" { + type = string +} + +# Application variables +variable "account_creator_id" { + default = "tmp_acount_creator.serhii.testnet" +} + +variable "external_signer_node_urls" { + type = list(string) + default = [] +} + +# Secrets +variable "account_creator_sk_secret_id" { + type = string +} + +variable "oidc_providers_secret_id" { + type = string +} + +variable "fast_auth_partners_secret_id" { + type = string +} + +variable "signer_configs" { + type = list(object({ + cipher_key_secret_id = string + sk_share_secret_id = string + })) +} + +variable "dev-connector" { + default = "projects/pagoda-shared-infrastructure/locations/us-east1/connectors/dev-connector1" +} + +variable "prod-connector" { + default = "projects/pagoda-shared-infrastructure/locations/us-east1/connectors/prod-us-east1-connector" +} + +data "google_compute_subnetwork" "dev_subnetwork" { + name = "dev-us-central1" + project = "pagoda-shared-infrastructure" + region = "us-central1" +} + +data "google_compute_subnetwork" "prod_subnetwork" { + name = "prod-us-central1" + project = "pagoda-shared-infrastructure" + region = "us-central1" +} + +data "google_compute_network" "dev_network" { + name = "dev" + project = "pagoda-shared-infrastructure" +} + +data "google_compute_network" "prod_network" { + name = "prod" + project = "pagoda-shared-infrastructure" +} + +variable "jwt_signature_pk_url" { + type = string +} + +variable "otlp_endpoint" { + type = string +} + +variable "opentelemetry_level" { + type = string +} \ No newline at end of file diff --git a/infra/partner/main.tf b/infra/partner/main.tf index f42377d78..ec25be594 100644 --- a/infra/partner/main.tf +++ b/infra/partner/main.tf @@ -21,6 +21,14 @@ provider "google" { zone = var.zone } +provider "docker" { + registry_auth { + address = "${var.region}-docker.pkg.dev" + username = "_json_key" + password = local.credentials + } +} + /* * Create brand new service account with basic IAM */ @@ -62,6 +70,31 @@ resource "google_secret_manager_secret_iam_member" "secret_share_secret_access" member = "serviceAccount:${google_service_account.service_account.email}" } +resource "google_secret_manager_secret_iam_member" "oidc_providers_secret_access" { + secret_id = var.oidc_providers_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +/* + * Create Artifact Registry repo, tag existing Docker image and push to the repo + */ +resource "google_artifact_registry_repository" "mpc_recovery" { + repository_id = "mpc-recovery-partner-${var.env}" + format = "DOCKER" +} + +resource "google_secret_manager_secret_iam_member" "secret_share_secret_access" { + secret_id = var.sk_share_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "docker_tag" "mpc_recovery" { + source_image = var.docker_image + target_image = "${var.region}-docker.pkg.dev/${var.project}/${google_artifact_registry_repository.mpc_recovery.name}/mpc-recovery-${var.env}" +} + /* * Create a partner signer node */ @@ -69,6 +102,7 @@ module "signer" { source = "../modules/signer" env = var.env + service_name = "partner-service-name" project = var.project region = var.region zone = var.zone @@ -79,11 +113,13 @@ module "signer" { cipher_key_secret_id = var.cipher_key_secret_id sk_share_secret_id = var.sk_share_secret_id - jwt_signature_pk_url = var.jwt_signature_pk_url + connector_id = var.connector_id depends_on = [ + docker_registry_image.mpc_recovery, google_secret_manager_secret_iam_member.cipher_key_secret_access, google_secret_manager_secret_iam_member.secret_share_secret_access, + google_secret_manager_secret_iam_member.oidc_providers_secret_access ] } diff --git a/infra/partner/template.tfvars b/infra/partner/template.tfvars index 008e28150..a9a77bf3b 100644 --- a/infra/partner/template.tfvars +++ b/infra/partner/template.tfvars @@ -3,7 +3,7 @@ project = "pagoda-discovery-platform-dev" region = "us-east1" zone = "us-east1-c" -docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev" +docker_image = "near/mpc-recovery" node_id = "0" oidc_providers_secret_id = "mpc-recovery-allowed-oidc-providers-0-dev" diff --git a/infra/partner/variables.tf b/infra/partner/variables.tf index 8eaee70c4..668b91ce8 100644 --- a/infra/partner/variables.tf +++ b/infra/partner/variables.tf @@ -20,6 +20,10 @@ variable "docker_image" { variable "node_id" { } +variable "connector_id" { + default = null +} + # Secrets variable "cipher_key_secret_id" { type = string @@ -29,6 +33,10 @@ variable "sk_share_secret_id" { type = string } -variable "jwt_signature_pk_url" { +variable "oidc_providers_secret_id" { type = string } + +variable "jwt_signature_pk_url" { + +} \ No newline at end of file