Internal LB (#321)

* feat: decouple secret management from terraform (#300) * initial commit for internal LB * added working dynamic TF for ILBs * separate prod and dev infra * changed connector id for dev * removed dev workflow, updated prod workflow * reflected dev and prod env * updated directory of dev workflow * updated all workflow dev directories * removed odic variable * updated prod workflow to have selectable networks * added var.env back to fix pipeline * reverted credential selection * made SA dynamic again * reverted GHA workflows * added back jwt signer url * updated prod env to include OTLP data * added correct naming convention for LB stuff * added correct credential info * added correct credential info --------- Co-authored-by: Daniyar Itegulov <ditegulov@gmail.com> Co-authored-by: kmaus-near <kody.maus.bluepisces@near.org> Co-authored-by: kmaus-near <109096383+kmaus-near@users.noreply.github.com>
near · Nov 15, 2023 · ef95741 · ef95741
1 parent 9b17acc
commit ef95741
Show file tree

Hide file tree

Showing 25 changed files with 810 additions and 20 deletions.
diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml
@@ -0,0 +1,152 @@
+name: Deploy to Prod environments.
+on:
+  workflow_dispatch:
+    inputs:
+      network:
+        type: choice
+        options:
+          - mainnet
+          - testnet
+        description: mainnet or testnet network
+        required: true
+      version:
+        description: What mainnet version number is this deployment? (e.g. v0.1.0)
+        required: true
+
+
+env:
+  PROJECT_PROD: "pagoda-discovery-platform-prod"
+  REGION: "us-east1"
+  IMAGE: us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery:${{ github.sha }}
+
+jobs:
+  build-mpc-recovery:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        name: "Checkout mpc-recovery"
+
+      - name: Login to Artifact Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGION }}-docker.pkg.dev
+          username: _json_key
+          password: ${{ secrets.GCP_CREDENTIALS_DEV }}
+
+      - name: Build Docker image and push to Google Artifact Registry
+        id: docker-push-tagged
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          file: ./Dockerfile
+          tags: "${{ env.IMAGE }}:${{ github.sha }}"
+
+  deploy-mpc-recovery-testnet:
+    runs-on: ubuntu-latest
+    if: github.event.inputs.network == 'testnet'
+    needs: build-mpc-recovery
+    env:
+      name: PROD
+    steps:
+      - uses: actions/checkout@v3
+        name: "Checkout mpc-recovery"
+
+      - name: "Authenticate to GCloud"
+        uses: "google-github-actions/auth@v1"
+        with:
+          credentials_json: "${{ secrets.GCP_CREDENTIALS_DEV }}"
+
+      - name: Deploy leader to Cloud Run Testnet
+        id: deploy-leader
+        uses: google-github-actions/deploy-cloudrun@v1
+        with:
+          image: "${{ env.IMAGE }}:${{ github.sha }}"
+          service: mpc-recovery-leader-testnet
+          region: us-east1
+          project_id: "${{ env.PROJECT_PROD }}"
+          tag: "${{ github.event.inputs.version }}"
+
+      - name: Deploy signer to Cloud Run testnet
+        id: deploy-signer-0-testnet
+        uses: google-github-actions/deploy-cloudrun@v1
+        with:
+          image: "${{ env.IMAGE }}:${{ github.sha }}"
+          service: mpc-recovery-signer-0-testnet
+          region: us-east1
+          project_id: "${{ env.PROJECT_PROD }}"
+          tag: "${{ github.event.inputs.version }}"
+
+      - name: Deploy signer to Cloud Run testnet
+        id: deploy-signer-1-testnet
+        uses: google-github-actions/deploy-cloudrun@v1
+        with:
+          image: "${{ env.IMAGE }}:${{ github.sha }}"
+          service: mpc-recovery-signer-1-testnet
+          region: us-east1
+          project_id: "${{ env.PROJECT_PROD }}"
+          tag: "${{ github.event.inputs.version }}"
+
+      - name: Deploy signer to Cloud Run testnet
+        id: deploy-signer-2-testnet
+        uses: google-github-actions/deploy-cloudrun@v1
+        with:
+          image: "${{ env.IMAGE }}:${{ github.sha }}"
+          service: mpc-recovery-signer-2-testnet
+          region: us-east1
+          project_id: "${{ env.PROJECT_PROD }}"
+          tag: "${{ github.event.inputs.version }}"
+
+  deploy-mpc-recovery-mainnet:
+    runs-on: ubuntu-latest
+    if: github.event.inputs.network == 'mainnet'
+    needs: build-mpc-recovery
+    env:
+      name: PROD
+    steps:
+      - uses: actions/checkout@v3
+        name: "Checkout mpc-recovery"
+
+      - name: "Authenticate to GCloud"
+        uses: "google-github-actions/auth@v1"
+        with:
+          credentials_json: "${{ secrets.GCP_CREDENTIALS_DEV }}"
+
+      - name: Deploy leader to Cloud Run mainnet
+        id: deploy-leader
+        uses: google-github-actions/deploy-cloudrun@v1
+        with:
+          image: "${{ env.IMAGE }}:${{ github.sha }}"
+          service: mpc-recovery-leader-mainnet
+          region: us-east1
+          project_id: "${{ env.PROJECT_PROD }}"
+          tag: "${{ github.event.inputs.version }}"
+
+      - name: Deploy signer to Cloud Run mainnet
+        id: deploy-signer-0
+        uses: google-github-actions/deploy-cloudrun@v1
+        with:
+          image: "${{ env.IMAGE }}:${{ github.sha }}"
+          service: mpc-recovery-signer-0-mainnet
+          region: us-east1
+          project_id: "${{ env.PROJECT_PROD }}"
+          tag: "${{ github.event.inputs.version }}"
+
+      - name: Deploy signer to Cloud Run mainnet
+        id: deploy-signer-1
+        uses: google-github-actions/deploy-cloudrun@v1
+        with:
+          image: "${{ env.IMAGE }}:${{ github.sha }}"
+          service: mpc-recovery-signer-1-mainnet
+          region: us-east1
+          project_id: "${{ env.PROJECT_PROD }}"
+          tag: "${{ github.event.inputs.version }}"
+
+      - name: Deploy signer to Cloud Run mainnet
+        id: deploy-signer-2
+        uses: google-github-actions/deploy-cloudrun@v1
+        with:
+          image: "${{ env.IMAGE }}:${{ github.sha }}"
+          service: mpc-recovery-signer-2-mainnet
+          region: us-east1
+          project_id: "${{ env.PROJECT_PROD }}"
+          tag: "${{ github.event.inputs.version }}"
diff --git a/.github/workflows/terraform-dev.yml b/.github/workflows/terraform-dev.yml
@@ -17,7 +17,7 @@ jobs:
       pull-requests: write
     defaults:
       run:
-        working-directory: ./infra
+        working-directory: ./infra/mpc-recovery-dev
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -54,6 +54,11 @@ jobs:
         env:
           GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }}
 
+      # Build Docker image.
+      - name: Docker Image
+        id: build
+        run: docker build .. -t near/mpc-recovery
+
       # Generates an execution plan for Terraform
       - name: Terraform Plan
         id: plan

diff --git a/.github/workflows/terraform-feature-env-destroy.yml b/.github/workflows/terraform-feature-env-destroy.yml
@@ -14,7 +14,7 @@ jobs:
       pull-requests: write
     defaults:
       run:
-        working-directory: ./infra
+        working-directory: ./infra/mpc-recovery-dev
     env:
       PR_NUMBER: ${{ github.event.number }}
     steps:

diff --git a/.github/workflows/terraform-feature-env.yml b/.github/workflows/terraform-feature-env.yml
@@ -15,7 +15,7 @@ jobs:
       checks: read
     defaults:
       run:
-        working-directory: ./infra
+        working-directory: ./infra/mpc-recovery-dev
     env:
       PR_NUMBER: ${{ github.event.number }}
     steps:

diff --git a/infra/README.md b/infra/README.md
@@ -0,0 +1,20 @@
+# MPC Recovery Infrastructure Overview
+
+There are currently 3 mostly static environments for MPC
+ - Mainnet (production)
+ - Testnet (production)
+ - Dev (development)
+
+ ## Mainnet/Testnet
+
+ Mainnet and Testnet infra code is in the directory `mpc-recovery-prod` and is built off of the `main` GitHub Branch
+   - This environment should be deployed via the GHA pipeline `deploy-prod.yml` manually in order to prevent unwanted changes
+   - Both Mainnet and Testnet are treated as production environments
+
+ ## Dev
+
+ The Dev environment infra code is located in the `mpc-recovery-dev` directory and is built off of the `develop` GitHub Branch
+   - This should be used as the main development environment
+   - Every time a pull request is opened up against the `develop` branch, a new, ephemeral environment is created with your changes
+     - *Note: These environments will have the associated PR number appended to all resources*
+   - When a pull request is approved and merged into the `develop` branch, a new revision is deployed to the static Dev environment with the PRs changes and the PRs ephemeral environment is destroyed
diff --git a/infra/modules/internal_cloudrun_lb/main.tf b/infra/modules/internal_cloudrun_lb/main.tf
@@ -0,0 +1,50 @@
+resource "google_compute_region_network_endpoint_group" "default_neg" {
+  name                  = "${var.name}-neg"
+  project               = var.project_id
+  network_endpoint_type = "SERVERLESS"
+  region                = var.region
+  cloud_run {
+    service = var.service_name
+  }
+}
+
+resource "google_compute_region_backend_service" "default" {
+  name                  = "${var.name}-backend-service"
+  project               = var.project_id
+  region                = var.region
+  protocol              = "HTTP"
+  load_balancing_scheme = "INTERNAL_MANAGED"
+  timeout_sec           = 30
+  backend {
+    group           = google_compute_region_network_endpoint_group.default_neg.id
+    balancing_mode  = "UTILIZATION"
+    capacity_scaler = 1.0
+  }
+}
+
+resource "google_compute_region_url_map" "default" {
+  name            = "${var.name}-url-map"
+  project         = var.project_id
+  region          = var.region
+  default_service = google_compute_region_backend_service.default.id
+}
+
+resource "google_compute_region_target_http_proxy" "default" {
+  name    = "${var.name}-http-proxy"
+  region  = var.region
+  project = var.project_id
+  url_map = google_compute_region_url_map.default.id
+}
+
+resource "google_compute_forwarding_rule" "default" {
+  name                  = "${var.name}-forwarding-rule"
+  project               = var.project_id
+  region                = var.region
+  ip_protocol           = "TCP"
+  load_balancing_scheme = "INTERNAL_MANAGED"
+  port_range            = "80"
+  target                = google_compute_region_target_http_proxy.default.id
+  network               = var.network_id
+  subnetwork            = var.subnetwork_id
+  network_tier          = "PREMIUM"
+}
diff --git a/infra/modules/internal_cloudrun_lb/variables.tf b/infra/modules/internal_cloudrun_lb/variables.tf
@@ -0,0 +1,29 @@
+variable "name" {
+  type        = string
+  description = "The name to use as prefix for load balancer resources."
+}
+
+variable "service_name" {
+  type        = string
+  description = "The cloud run service name"
+}
+
+variable "project_id" {
+  type        = string
+  description = "The GCP project these resources belong to"
+}
+
+variable "region" {
+  type        = string
+  description = "The region where resources will live."
+}
+
+variable "network_id" {
+  type        = string
+  description = "The VPC network to connect to."
+}
+
+variable "subnetwork_id" {
+  type        = string
+  description = "Subnet for hosting the load balancer."
+}
diff --git a/infra/modules/leader/main.tf b/infra/modules/leader/main.tf
@@ -1,11 +1,18 @@
 resource "google_cloud_run_v2_service" "leader" {
-  name     = "mpc-recovery-leader-${var.env}"
+  name     = var.service_name
   location = var.region
   ingress  = "INGRESS_TRAFFIC_ALL"
 
   template {
     service_account = var.service_account_email
 
+    annotations = var.metadata_annotations == null ? null : var.metadata_annotations
+
+    vpc_access {
+      connector = var.connector_id
+      egress    = "PRIVATE_RANGES_ONLY"
+    }
+
     scaling {
       min_instance_count = 1
       max_instance_count = 1
@@ -48,7 +55,7 @@ resource "google_cloud_run_v2_service" "leader" {
         value_source {
           secret_key_ref {
             secret  = var.account_creator_sk_secret_id
-            version = "1"
+            version = "latest"
           }
         }
       }
@@ -82,7 +89,6 @@ resource "google_cloud_run_v2_service" "leader" {
       ports {
         container_port = 3000
       }
-
       resources {
         cpu_idle = false
 

diff --git a/infra/modules/leader/variables.tf b/infra/modules/leader/variables.tf
@@ -6,6 +6,16 @@ variable "project" {
   type = string
 }
 
+variable "connector_id" {
+  description = "VPC connector ID for internal traffic"
+}
+
+variable "metadata_annotations" {
+  type        = map(any)
+  default     = null
+  description = "Annotations for the metadata associated with this Service."
+}
+
 variable "region" {
   type = string
 }
@@ -48,6 +58,9 @@ variable "fast_auth_partners_secret_id" {
   type = string
 }
 
+variable "service_name" {
+  type = string
+}
 variable "jwt_signature_pk_url" {
   type = string
 }

diff --git a/infra/modules/signer/main.tf b/infra/modules/signer/main.tf
@@ -1,11 +1,18 @@
 resource "google_cloud_run_v2_service" "signer" {
-  name     = "mpc-recovery-signer-${var.node_id}-${var.env}"
+  name     = var.service_name
   location = var.region
   ingress  = "INGRESS_TRAFFIC_ALL"
 
   template {
     service_account = var.service_account_email
 
+    annotations = var.metadata_annotations == null ? null : var.metadata_annotations
+
+    vpc_access {
+      connector = var.connector_id == null ? null : var.connector_id
+      egress    = "PRIVATE_RANGES_ONLY"
+    }
+
     scaling {
       min_instance_count = 1
       max_instance_count = 1
@@ -62,6 +69,8 @@ resource "google_cloud_run_v2_service" "signer" {
         container_port = 3000
       }
 
+
+
       resources {
         cpu_idle = false