From f860cbe701924a4974942ac79652ca44a1ef0bd7 Mon Sep 17 00:00:00 2001 From: Aarushi Date: Wed, 16 Oct 2024 12:39:31 +0100 Subject: [PATCH 01/13] ci with workload identity --- .../workflows/platform-autogpt-deploy.yaml | 151 ++++++++++++++++++ .../helm/autogpt-builder/values.dev.yaml | 4 +- .../infra/helm/autogpt-server/values.dev.yaml | 4 +- .../helm/autogpt-server/values.prod.yaml | 4 +- .../autogpt-websocket-server/values.dev.yaml | 2 +- .../infra/terraform/environments/dev.tfvars | 46 +++++- autogpt_platform/infra/terraform/main.tf | 1 + .../infra/terraform/modules/iam/main.tf | 27 ++++ .../infra/terraform/modules/iam/outputs.tf | 12 +- .../infra/terraform/modules/iam/variables.tf | 13 ++ autogpt_platform/infra/terraform/variables.tf | 16 ++ 11 files changed, 268 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/platform-autogpt-deploy.yaml diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml new file mode 100644 index 000000000000..1df5408f342a --- /dev/null +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -0,0 +1,151 @@ +name: AutoGPT Platform - Build, Push, and Deploy Dev + +on: + workflow_dispatch: + push: + branches: [ dev ] + paths: + - 'autogpt_platform/backend/**' + - 'autogpt_platform/frontend/**' + - 'autogpt_platform/market/**' + +permissions: + contents: 'read' + id-token: 'write' + +env: + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + GKE_CLUSTER: dev-gke-cluster + GKE_ZONE: us-central1-a + NAMESPACE: dev-agpt + +jobs: + build-push-deploy: + name: Build, Push, and Deploy + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - id: 'auth' + uses: 'google-github-actions/auth@v1' + with: + workload_identity_provider: 'projects/agpt-dev/locations/global/workloadIdentityPools/dev-pool/providers/github' + service_account: 'dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + + - name: 'Configure Docker' + run: | + gcloud auth configure-docker gcr.io + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Check for changes + id: check_changes + run: | + git fetch origin dev + BACKEND_CHANGED=$(git diff --name-only origin/dev HEAD | grep "^autogpt_platform/backend/" && echo "true" || echo "false") + FRONTEND_CHANGED=$(git diff --name-only origin/dev HEAD | grep "^autogpt_platform/frontend/" && echo "true" || echo "false") + MARKET_CHANGED=$(git diff --name-only origin/dev HEAD | grep "^autogpt_platform/market/" && echo "true" || echo "false") + echo "backend_changed=$BACKEND_CHANGED" >> $GITHUB_OUTPUT + echo "frontend_changed=$FRONTEND_CHANGED" >> $GITHUB_OUTPUT + echo "market_changed=$MARKET_CHANGED" >> $GITHUB_OUTPUT + + - name: Get GKE credentials + uses: 'google-github-actions/get-gke-credentials@v1' + with: + cluster_name: ${{ env.GKE_CLUSTER }} + location: ${{ env.GKE_ZONE }} + + - name: Build and Push Backend + if: steps.check_changes.outputs.backend_changed == 'true' + uses: docker/build-push-action@v2 + with: + context: ./autogpt_platform + file: ./autogpt_platform/backend/Dockerfile + push: true + tags: gcr.io/${{ env.PROJECT_ID }}/autogpt-backend:${{ github.sha }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Build and Push Frontend + if: steps.check_changes.outputs.frontend_changed == 'true' + uses: docker/build-push-action@v2 + with: + context: ./autogpt_platform + file: ./autogpt_platform/frontend/Dockerfile + push: true + tags: gcr.io/${{ env.PROJECT_ID }}/autogpt-frontend:${{ github.sha }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Build and Push Market + if: steps.check_changes.outputs.market_changed == 'true' + uses: docker/build-push-action@v2 + with: + context: ./autogpt_platform + file: ./autogpt_platform/market/Dockerfile + push: true + tags: gcr.io/${{ env.PROJECT_ID }}/autogpt-market:${{ github.sha }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: v3.4.0 + + - name: Deploy Backend + if: steps.check_changes.outputs.backend_changed == 'true' + run: | + helm upgrade autogpt-server ./autogpt-server \ + --namespace ${{ env.NAMESPACE }} \ + -f autogpt-server/values.yaml \ + -f autogpt-server/values.dev.yaml \ + --set image.tag=${{ github.sha }} + + - name: Deploy Websocket + if: steps.check_changes.outputs.backend_changed == 'true' + run: | + helm upgrade autogpt-websocket-server ./autogpt-websocket-server \ + --namespace ${{ env.NAMESPACE }} \ + -f autogpt-websocket-server/values.yaml \ + -f autogpt-websocket-server/values.dev.yaml \ + --set image.tag=${{ github.sha }} + + - name: Deploy Market + if: steps.check_changes.outputs.market_changed == 'true' + run: | + helm upgrade autogpt-market ./autogpt-market \ + --namespace ${{ env.NAMESPACE }} \ + -f autogpt-market/values.yaml \ + -f autogpt-market/values.dev.yaml \ + --set image.tag=${{ github.sha }} + + - name: Deploy Frontend + if: steps.check_changes.outputs.frontend_changed == 'true' + run: | + helm upgrade autogpt-builder ./autogpt-builder \ + --namespace ${{ env.NAMESPACE }} \ + -f autogpt-builder/values.yaml \ + -f autogpt-builder/values.dev.yaml \ + --set image.tag=${{ github.sha }} \ No newline at end of file diff --git a/autogpt_platform/infra/helm/autogpt-builder/values.dev.yaml b/autogpt_platform/infra/helm/autogpt-builder/values.dev.yaml index 1821acc24a39..128ea3ee44a5 100644 --- a/autogpt_platform/infra/helm/autogpt-builder/values.dev.yaml +++ b/autogpt_platform/infra/helm/autogpt-builder/values.dev.yaml @@ -1,9 +1,9 @@ # dev values, overwrite base values as needed. image: - repository: us-east1-docker.pkg.dev/agpt-dev/agpt-builder-dev/agpt-builder-dev + repository: us-east1-docker.pkg.dev/agpt-dev/agpt-frontend-dev/agpt-frontend-dev pullPolicy: Always - tag: "fe3d2a9" + tag: "latest" serviceAccount: annotations: diff --git a/autogpt_platform/infra/helm/autogpt-server/values.dev.yaml b/autogpt_platform/infra/helm/autogpt-server/values.dev.yaml index 19349017aef4..b7488e0fd078 100644 --- a/autogpt_platform/infra/helm/autogpt-server/values.dev.yaml +++ b/autogpt_platform/infra/helm/autogpt-server/values.dev.yaml @@ -1,7 +1,7 @@ # dev values, overwrite base values as needed. image: - repository: us-east1-docker.pkg.dev/agpt-dev/agpt-server-dev/agpt-server-dev + repository: us-east1-docker.pkg.dev/agpt-dev/agpt-backend-dev/agpt-backend-dev pullPolicy: Always tag: "latest" @@ -58,7 +58,7 @@ resources: livenessProbe: httpGet: - path: /heath + path: /health port: 8006 initialDelaySeconds: 30 periodSeconds: 10 diff --git a/autogpt_platform/infra/helm/autogpt-server/values.prod.yaml b/autogpt_platform/infra/helm/autogpt-server/values.prod.yaml index 0e73afa33175..eb314a899d1b 100644 --- a/autogpt_platform/infra/helm/autogpt-server/values.prod.yaml +++ b/autogpt_platform/infra/helm/autogpt-server/values.prod.yaml @@ -72,7 +72,7 @@ cors: livenessProbe: httpGet: - path: /heath + path: /health port: 8006 initialDelaySeconds: 30 periodSeconds: 10 @@ -80,7 +80,7 @@ livenessProbe: failureThreshold: 6 readinessProbe: httpGet: - path: /heath + path: /health port: 8006 initialDelaySeconds: 30 periodSeconds: 10 diff --git a/autogpt_platform/infra/helm/autogpt-websocket-server/values.dev.yaml b/autogpt_platform/infra/helm/autogpt-websocket-server/values.dev.yaml index a977f9bece0c..dd26b32f4030 100644 --- a/autogpt_platform/infra/helm/autogpt-websocket-server/values.dev.yaml +++ b/autogpt_platform/infra/helm/autogpt-websocket-server/values.dev.yaml @@ -1,7 +1,7 @@ replicaCount: 1 # not scaling websocket server for now image: - repository: us-east1-docker.pkg.dev/agpt-dev/agpt-server-dev/agpt-server-dev + repository: us-east1-docker.pkg.dev/agpt-dev/agpt-backend-dev/agpt-backend-dev tag: latest pullPolicy: Always diff --git a/autogpt_platform/infra/terraform/environments/dev.tfvars b/autogpt_platform/infra/terraform/environments/dev.tfvars index 1f72c6d907a0..a1fba47c7601 100644 --- a/autogpt_platform/infra/terraform/environments/dev.tfvars +++ b/autogpt_platform/infra/terraform/environments/dev.tfvars @@ -28,6 +28,10 @@ service_accounts = { "dev-agpt-market-sa" = { display_name = "AutoGPT Dev Market Server Account" description = "Service account for agpt dev market server" + }, + "dev-github-actions-sa" = { + display_name = "GitHub Actions Dev Service Account" + description = "Service account for GitHub Actions deployments to dev" } } @@ -51,6 +55,11 @@ workload_identity_bindings = { service_account_name = "dev-agpt-market-sa" namespace = "dev-agpt" ksa_name = "dev-agpt-market-sa" + }, + "dev-github-actions-workload-identity" = { + service_account_name = "dev-github-actions-sa" + namespace = "dev-agpt" + ksa_name = "dev-github-actions-sa" } } @@ -59,7 +68,8 @@ role_bindings = { "serviceAccount:dev-agpt-server-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-builder-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-ws-server-sa@agpt-dev.iam.gserviceaccount.com", - "serviceAccount:dev-agpt-market-sa@agpt-dev.iam.gserviceaccount.com" + "serviceAccount:dev-agpt-market-sa@agpt-dev.iam.gserviceaccount.com", + "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" ], "roles/cloudsql.client" = [ "serviceAccount:dev-agpt-server-sa@agpt-dev.iam.gserviceaccount.com", @@ -80,7 +90,8 @@ role_bindings = { "serviceAccount:dev-agpt-server-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-builder-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-ws-server-sa@agpt-dev.iam.gserviceaccount.com", - "serviceAccount:dev-agpt-market-sa@agpt-dev.iam.gserviceaccount.com" + "serviceAccount:dev-agpt-market-sa@agpt-dev.iam.gserviceaccount.com", + "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" ] "roles/compute.networkUser" = [ "serviceAccount:dev-agpt-server-sa@agpt-dev.iam.gserviceaccount.com", @@ -93,7 +104,13 @@ role_bindings = { "serviceAccount:dev-agpt-builder-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-ws-server-sa@agpt-dev.iam.gserviceaccount.com", "serviceAccount:dev-agpt-market-sa@agpt-dev.iam.gserviceaccount.com" - ] + ], + "roles/artifactregistry.writer" = [ + "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" + ], + "roles/container.viewer" = [ + "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" + ], } pods_ip_cidr_range = "10.1.0.0/16" @@ -101,4 +118,25 @@ services_ip_cidr_range = "10.2.0.0/20" public_bucket_names = ["website-artifacts"] standard_bucket_names = [] -bucket_admins = ["gcp-devops-agpt@agpt.co", "gcp-developers@agpt.co"] \ No newline at end of file +bucket_admins = ["gcp-devops-agpt@agpt.co", "gcp-developers@agpt.co"] + +workload_identity_pools = { + "dev-pool" = { + display_name = "Development Identity Pool" + providers = { + "github" = { + issuer_uri = "https://token.actions.githubusercontent.com" + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.repository" = "assertion.repository" + "attribute.repository_owner" = "assertion.repository_owner" + } + } + } + service_accounts = { + "dev-github-actions-sa" = [ + "Significant-Gravitas/AutoGPT" + ] + } + } +} \ No newline at end of file diff --git a/autogpt_platform/infra/terraform/main.tf b/autogpt_platform/infra/terraform/main.tf index 31049ed40cb9..047ebd59c661 100644 --- a/autogpt_platform/infra/terraform/main.tf +++ b/autogpt_platform/infra/terraform/main.tf @@ -61,6 +61,7 @@ module "iam" { service_accounts = var.service_accounts workload_identity_bindings = var.workload_identity_bindings role_bindings = var.role_bindings + workload_identity_pools = var.workload_identity_pools } module "storage" { diff --git a/autogpt_platform/infra/terraform/modules/iam/main.tf b/autogpt_platform/infra/terraform/modules/iam/main.tf index 3a07d6926456..f632f4c42b43 100644 --- a/autogpt_platform/infra/terraform/modules/iam/main.tf +++ b/autogpt_platform/infra/terraform/modules/iam/main.tf @@ -23,4 +23,31 @@ resource "google_project_iam_binding" "role_bindings" { role = each.key members = each.value +} + +resource "google_iam_workload_identity_pool" "pools" { + for_each = var.workload_identity_pools + workload_identity_pool_id = each.key + display_name = each.value.display_name +} + +resource "google_iam_workload_identity_pool_provider" "providers" { + for_each = merge([ + for pool_id, pool in var.workload_identity_pools : { + for provider_id, provider in pool.providers : + "${pool_id}/${provider_id}" => merge(provider, { + pool_id = pool_id + }) + } + ]...) + + workload_identity_pool_id = split("/", each.key)[0] + workload_identity_pool_provider_id = split("/", each.key)[1] + + attribute_mapping = each.value.attribute_mapping + oidc { + issuer_uri = each.value.issuer_uri + allowed_audiences = each.value.allowed_audiences + } + attribute_condition = "assertion.repository_owner==\"Significant-Gravitas\"" } \ No newline at end of file diff --git a/autogpt_platform/infra/terraform/modules/iam/outputs.tf b/autogpt_platform/infra/terraform/modules/iam/outputs.tf index b503414873a9..19354364bebb 100644 --- a/autogpt_platform/infra/terraform/modules/iam/outputs.tf +++ b/autogpt_platform/infra/terraform/modules/iam/outputs.tf @@ -1,4 +1,14 @@ output "service_account_emails" { description = "The emails of the created service accounts" value = { for k, v in google_service_account.service_accounts : k => v.email } -} \ No newline at end of file +} + +output "workload_identity_pools" { + value = google_iam_workload_identity_pool.pools +} + +output "workload_identity_providers" { + value = { + for k, v in google_iam_workload_identity_pool_provider.providers : k => v.name + } +} diff --git a/autogpt_platform/infra/terraform/modules/iam/variables.tf b/autogpt_platform/infra/terraform/modules/iam/variables.tf index 61637a718783..c9563ea0c7b8 100644 --- a/autogpt_platform/infra/terraform/modules/iam/variables.tf +++ b/autogpt_platform/infra/terraform/modules/iam/variables.tf @@ -26,4 +26,17 @@ variable "role_bindings" { description = "Map of roles to list of members" type = map(list(string)) default = {} +} + +variable "workload_identity_pools" { + type = map(object({ + display_name = string + providers = map(object({ + issuer_uri = string + attribute_mapping = map(string) + allowed_audiences = optional(list(string)) + })) + service_accounts = map(list(string)) # Map of SA to list of allowed principals + })) + default = {} } \ No newline at end of file diff --git a/autogpt_platform/infra/terraform/variables.tf b/autogpt_platform/infra/terraform/variables.tf index 1afd9509fcbc..3b4eb92dff9c 100644 --- a/autogpt_platform/infra/terraform/variables.tf +++ b/autogpt_platform/infra/terraform/variables.tf @@ -130,3 +130,19 @@ variable "bucket_admins" { default = ["gcp-devops-agpt@agpt.co", "gcp-developers@agpt.co"] } +variable "workload_identity_pools" { + type = map(object({ + display_name = string + providers = map(object({ + issuer_uri = string + attribute_mapping = map(string) + allowed_audiences = optional(list(string)) + })) + service_accounts = map(list(string)) + })) + default = {} + description = "Configuration for workload identity pools and their providers" +} + + + From 2b17c7716908c0a857efb140e12164e898b456d2 Mon Sep 17 00:00:00 2001 From: Aarushi Date: Sun, 20 Oct 2024 12:51:05 +0100 Subject: [PATCH 02/13] temp update --- .github/workflows/platform-autogpt-deploy.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index 1df5408f342a..41a461bef73a 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -3,8 +3,9 @@ name: AutoGPT Platform - Build, Push, and Deploy Dev on: workflow_dispatch: push: - branches: [ dev ] + branches: [ dev, aarushikansal/secrt-944-set-up-deploys-from-dev ] paths: + - '*' - 'autogpt_platform/backend/**' - 'autogpt_platform/frontend/**' - 'autogpt_platform/market/**' From de51b1455fc2ad2f48268934f988769ad73e2859 Mon Sep 17 00:00:00 2001 From: Aarushi Date: Sun, 20 Oct 2024 12:53:15 +0100 Subject: [PATCH 03/13] update name --- .github/workflows/platform-autogpt-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index 41a461bef73a..433c74e37120 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -1,4 +1,4 @@ -name: AutoGPT Platform - Build, Push, and Deploy Dev +name: AutoGPT Platform - Build, Push, and Deploy Dev Environment on: workflow_dispatch: From 6addbb8581810e35f8758ce5766e5c978a86c2d1 Mon Sep 17 00:00:00 2001 From: Aarushi Date: Sun, 20 Oct 2024 12:58:17 +0100 Subject: [PATCH 04/13] wip --- .github/workflows/platform-autogpt-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index 433c74e37120..67732f123fa1 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -5,7 +5,7 @@ on: push: branches: [ dev, aarushikansal/secrt-944-set-up-deploys-from-dev ] paths: - - '*' + - '**' - 'autogpt_platform/backend/**' - 'autogpt_platform/frontend/**' - 'autogpt_platform/market/**' From 23065a0d7d3aee420225d6403a4bd17964c6fcf3 Mon Sep 17 00:00:00 2001 From: Aarushi Date: Sun, 20 Oct 2024 13:02:33 +0100 Subject: [PATCH 05/13] update auth step --- .github/workflows/platform-autogpt-deploy.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index 67732f123fa1..60458cd80980 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -34,8 +34,11 @@ jobs: - id: 'auth' uses: 'google-github-actions/auth@v1' with: - workload_identity_provider: 'projects/agpt-dev/locations/global/workloadIdentityPools/dev-pool/providers/github' + workload_identity_provider: 'projects/agpt-dev/locations/global/workloadIdentityPools/dev-pool/providers/github-identity-provider' service_account: 'dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com' + token_format: 'access_token' + create_credentials_file: true + audience: 'https://container.googleapis.com/v1/projects/agpt-dev/locations/us-central1-a/clusters/dev-gke-cluster' - name: 'Set up Cloud SDK' uses: 'google-github-actions/setup-gcloud@v1' From 8b5fc1fcdcdfb4edf53240835f469bae5f00ad20 Mon Sep 17 00:00:00 2001 From: Aarushi Date: Sun, 20 Oct 2024 13:24:52 +0100 Subject: [PATCH 06/13] update provider name --- .github/workflows/platform-autogpt-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index 60458cd80980..c024d64f6764 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -34,7 +34,7 @@ jobs: - id: 'auth' uses: 'google-github-actions/auth@v1' with: - workload_identity_provider: 'projects/agpt-dev/locations/global/workloadIdentityPools/dev-pool/providers/github-identity-provider' + workload_identity_provider: 'projects/638488734936/locations/global/workloadIdentityPools/dev-pool/providers/github' service_account: 'dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com' token_format: 'access_token' create_credentials_file: true From 172ccb730bebc5f015621b7b72daedc22867d037 Mon Sep 17 00:00:00 2001 From: Aarushi Date: Sun, 20 Oct 2024 13:27:07 +0100 Subject: [PATCH 07/13] remove audience --- .github/workflows/platform-autogpt-deploy.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index c024d64f6764..57a0c0665155 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -38,7 +38,6 @@ jobs: service_account: 'dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com' token_format: 'access_token' create_credentials_file: true - audience: 'https://container.googleapis.com/v1/projects/agpt-dev/locations/us-central1-a/clusters/dev-gke-cluster' - name: 'Set up Cloud SDK' uses: 'google-github-actions/setup-gcloud@v1' From c039ae2a1306a568cf0bd6759c46da6f7fe390cf Mon Sep 17 00:00:00 2001 From: Aarushi Date: Sun, 20 Oct 2024 13:42:11 +0100 Subject: [PATCH 08/13] temp set to false --- .github/workflows/platform-autogpt-deploy.yaml | 4 ++-- autogpt_platform/infra/terraform/environments/dev.tfvars | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index 57a0c0665155..d47828d73079 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -75,7 +75,7 @@ jobs: location: ${{ env.GKE_ZONE }} - name: Build and Push Backend - if: steps.check_changes.outputs.backend_changed == 'true' + if: steps.check_changes.outputs.backend_changed == 'false' uses: docker/build-push-action@v2 with: context: ./autogpt_platform @@ -97,7 +97,7 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - name: Build and Push Market - if: steps.check_changes.outputs.market_changed == 'true' + if: steps.check_changes.outputs.market_changed == 'false' uses: docker/build-push-action@v2 with: context: ./autogpt_platform diff --git a/autogpt_platform/infra/terraform/environments/dev.tfvars b/autogpt_platform/infra/terraform/environments/dev.tfvars index a1fba47c7601..c193e8399a9a 100644 --- a/autogpt_platform/infra/terraform/environments/dev.tfvars +++ b/autogpt_platform/infra/terraform/environments/dev.tfvars @@ -111,6 +111,10 @@ role_bindings = { "roles/container.viewer" = [ "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" ], + "roles/iam.serviceAccountTokenCreator" = [ + "principalSet://iam.googleapis.com/projects/638488734936/locations/global/workloadIdentityPools/dev-pool/*", + "serviceAccount:dev-github-actions-sa@agpt-dev.iam.gserviceaccount.com" + ] } pods_ip_cidr_range = "10.1.0.0/16" From 68c1735b165335e5f8caef70138b540b1785c822 Mon Sep 17 00:00:00 2001 From: Aarushi Date: Sun, 20 Oct 2024 13:45:17 +0100 Subject: [PATCH 09/13] update registry naming --- .github/workflows/platform-autogpt-deploy.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index d47828d73079..5bad4351d586 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -81,7 +81,7 @@ jobs: context: ./autogpt_platform file: ./autogpt_platform/backend/Dockerfile push: true - tags: gcr.io/${{ env.PROJECT_ID }}/autogpt-backend:${{ github.sha }} + tags: us-east1-docker.pkg.dev/agpt-dev/agpt-backend-dev/agpt-backend-dev:${{ github.sha }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max @@ -92,7 +92,7 @@ jobs: context: ./autogpt_platform file: ./autogpt_platform/frontend/Dockerfile push: true - tags: gcr.io/${{ env.PROJECT_ID }}/autogpt-frontend:${{ github.sha }} + tags: us-east1-docker.pkg.dev/agpt-dev/agpt-frontend-dev/agpt-frontend-dev:${{ github.sha }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max @@ -103,7 +103,7 @@ jobs: context: ./autogpt_platform file: ./autogpt_platform/market/Dockerfile push: true - tags: gcr.io/${{ env.PROJECT_ID }}/autogpt-market:${{ github.sha }} + tags: us-east1-docker.pkg.dev/agpt-dev/agpt-market-dev/agpt-market-dev:${{ github.sha }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max From 6d3f7c59c2160bf599cee78b354f7d3085507ae4 Mon Sep 17 00:00:00 2001 From: Aarushi Date: Sun, 20 Oct 2024 13:49:35 +0100 Subject: [PATCH 10/13] update context --- .github/workflows/platform-autogpt-deploy.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index 5bad4351d586..42a1707d8dba 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -78,7 +78,7 @@ jobs: if: steps.check_changes.outputs.backend_changed == 'false' uses: docker/build-push-action@v2 with: - context: ./autogpt_platform + context: . file: ./autogpt_platform/backend/Dockerfile push: true tags: us-east1-docker.pkg.dev/agpt-dev/agpt-backend-dev/agpt-backend-dev:${{ github.sha }} @@ -89,7 +89,7 @@ jobs: if: steps.check_changes.outputs.frontend_changed == 'true' uses: docker/build-push-action@v2 with: - context: ./autogpt_platform + context: . file: ./autogpt_platform/frontend/Dockerfile push: true tags: us-east1-docker.pkg.dev/agpt-dev/agpt-frontend-dev/agpt-frontend-dev:${{ github.sha }} @@ -100,7 +100,7 @@ jobs: if: steps.check_changes.outputs.market_changed == 'false' uses: docker/build-push-action@v2 with: - context: ./autogpt_platform + context: . file: ./autogpt_platform/market/Dockerfile push: true tags: us-east1-docker.pkg.dev/agpt-dev/agpt-market-dev/agpt-market-dev:${{ github.sha }} From c422e8a64845544e63f625dc0f0f96e0110b9db7 Mon Sep 17 00:00:00 2001 From: Aarushi Date: Sun, 20 Oct 2024 23:31:58 +0100 Subject: [PATCH 11/13] update login --- .github/workflows/platform-autogpt-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index 42a1707d8dba..697e5e315371 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -44,7 +44,7 @@ jobs: - name: 'Configure Docker' run: | - gcloud auth configure-docker gcr.io + gcloud auth configure-docker us-east1-docker.pkg.dev - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 From 401eff026c78993f0f771336f9094894ad80d1ad Mon Sep 17 00:00:00 2001 From: Aarushi Date: Mon, 21 Oct 2024 10:27:05 +0100 Subject: [PATCH 12/13] revert temp updates --- .github/workflows/platform-autogpt-deploy.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/platform-autogpt-deploy.yaml b/.github/workflows/platform-autogpt-deploy.yaml index 697e5e315371..97c2fe78749e 100644 --- a/.github/workflows/platform-autogpt-deploy.yaml +++ b/.github/workflows/platform-autogpt-deploy.yaml @@ -1,11 +1,9 @@ name: AutoGPT Platform - Build, Push, and Deploy Dev Environment on: - workflow_dispatch: push: - branches: [ dev, aarushikansal/secrt-944-set-up-deploys-from-dev ] + branches: [ dev ] paths: - - '**' - 'autogpt_platform/backend/**' - 'autogpt_platform/frontend/**' - 'autogpt_platform/market/**' @@ -75,7 +73,7 @@ jobs: location: ${{ env.GKE_ZONE }} - name: Build and Push Backend - if: steps.check_changes.outputs.backend_changed == 'false' + if: steps.check_changes.outputs.backend_changed == 'true' uses: docker/build-push-action@v2 with: context: . @@ -97,7 +95,7 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - name: Build and Push Market - if: steps.check_changes.outputs.market_changed == 'false' + if: steps.check_changes.outputs.market_changed == 'true' uses: docker/build-push-action@v2 with: context: . From c2765d150681b18dc7948c8e66aad0bb349217ac Mon Sep 17 00:00:00 2001 From: Aarushi Date: Mon, 21 Oct 2024 11:11:37 +0100 Subject: [PATCH 13/13] add prod iam and pool --- .../infra/terraform/environments/prod.tfvars | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/autogpt_platform/infra/terraform/environments/prod.tfvars b/autogpt_platform/infra/terraform/environments/prod.tfvars index e9351389285a..4bceda49957a 100644 --- a/autogpt_platform/infra/terraform/environments/prod.tfvars +++ b/autogpt_platform/infra/terraform/environments/prod.tfvars @@ -28,6 +28,11 @@ service_accounts = { "prod-agpt-market-sa" = { display_name = "AutoGPT prod Market backend Account" description = "Service account for agpt prod market backend" + }, + "prod-github-actions-workload-identity" = { + service_account_name = "prod-github-actions-sa" + namespace = "prod-agpt" + ksa_name = "prod-github-actions-sa" } } @@ -59,7 +64,8 @@ role_bindings = { "serviceAccount:prod-agpt-backend-sa@agpt-prod.iam.gserviceaccount.com", "serviceAccount:prod-agpt-frontend-sa@agpt-prod.iam.gserviceaccount.com", "serviceAccount:prod-agpt-ws-backend-sa@agpt-prod.iam.gserviceaccount.com", - "serviceAccount:prod-agpt-market-sa@agpt-prod.iam.gserviceaccount.com" + "serviceAccount:prod-agpt-market-sa@agpt-prod.iam.gserviceaccount.com", + "serviceAccount:prod-github-actions-sa@agpt-prod.iam.gserviceaccount.com" ], "roles/cloudsql.client" = [ "serviceAccount:prod-agpt-backend-sa@agpt-prod.iam.gserviceaccount.com", @@ -80,7 +86,8 @@ role_bindings = { "serviceAccount:prod-agpt-backend-sa@agpt-prod.iam.gserviceaccount.com", "serviceAccount:prod-agpt-frontend-sa@agpt-prod.iam.gserviceaccount.com", "serviceAccount:prod-agpt-ws-backend-sa@agpt-prod.iam.gserviceaccount.com", - "serviceAccount:prod-agpt-market-sa@agpt-prod.iam.gserviceaccount.com" + "serviceAccount:prod-agpt-market-sa@agpt-prod.iam.gserviceaccount.com", + "serviceAccount:prod-github-actions-sa@agpt-prod.iam.gserviceaccount.com" ] "roles/compute.networkUser" = [ "serviceAccount:prod-agpt-backend-sa@agpt-prod.iam.gserviceaccount.com", @@ -93,6 +100,16 @@ role_bindings = { "serviceAccount:prod-agpt-frontend-sa@agpt-prod.iam.gserviceaccount.com", "serviceAccount:prod-agpt-ws-backend-sa@agpt-prod.iam.gserviceaccount.com", "serviceAccount:prod-agpt-market-sa@agpt-prod.iam.gserviceaccount.com" + ], + "roles/artifactregistry.writer" = [ + "serviceAccount:prod-github-actions-sa@agpt-prod.iam.gserviceaccount.com" + ], + "roles/container.viewer" = [ + "serviceAccount:prod-github-actions-sa@agpt-prod.iam.gserviceaccount.com" + ], + "roles/iam.serviceAccountTokenCreator" = [ + "principalSet://iam.googleapis.com/projects/638488734936/locations/global/workloadIdentityPools/prod-pool/*", + "serviceAccount:prod-github-actions-sa@agpt-prod.iam.gserviceaccount.com" ] } @@ -101,4 +118,25 @@ services_ip_cidr_range = "10.2.0.0/20" public_bucket_names = ["website-artifacts"] standard_bucket_names = [] -bucket_admins = ["gcp-devops-agpt@agpt.co", "gcp-developers@agpt.co"] \ No newline at end of file +bucket_admins = ["gcp-devops-agpt@agpt.co", "gcp-developers@agpt.co"] + +workload_identity_pools = { + "dev-pool" = { + display_name = "Production Identity Pool" + providers = { + "github" = { + issuer_uri = "https://token.actions.githubusercontent.com" + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.repository" = "assertion.repository" + "attribute.repository_owner" = "assertion.repository_owner" + } + } + } + service_accounts = { + "prod-github-actions-sa" = [ + "Significant-Gravitas/AutoGPT" + ] + } + } +} \ No newline at end of file