feat(platform/ci) Set up deploys from dev (#8355)

* ci with workload identity

* temp update

* update name

* wip

* update auth step

* update provider name

* remove audience

* temp set to false

* update registry naming

* update context

* update login

* revert temp updates

.github/workflows/platform-autogpt-deploy.yaml

@@ -0,0 +1,152 @@
+name: AutoGPT Platform - Build, Push, and Deploy Dev Environment
+
+on:
+  push:
+    branches: [ dev ]
+    paths:
+      - 'autogpt_platform/backend/**'
+      - 'autogpt_platform/frontend/**'
+      - 'autogpt_platform/market/**'
+
+permissions:
+  contents: 'read'
+  id-token: 'write'
+
+env:
+  PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+  GKE_CLUSTER: dev-gke-cluster
+  GKE_ZONE: us-central1-a
+  NAMESPACE: dev-agpt
+
+jobs:
+  build-push-deploy:
+    name: Build, Push, and Deploy
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - id: 'auth'
+      uses: 'google-github-actions/auth@v1'
+      with:
+        workload_identity_provider: 'projects/638488734936/locations/global/workloadIdentityPools/dev-pool/providers/github'
+        service_account: '[email protected]'
+        token_format: 'access_token'
+        create_credentials_file: true
+
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v1'
+
+    - name: 'Configure Docker'
+      run: |
+        gcloud auth configure-docker us-east1-docker.pkg.dev
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Cache Docker layers
+      uses: actions/cache@v2
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
+
+    - name: Check for changes
+      id: check_changes
+      run: |
+        git fetch origin dev
+        BACKEND_CHANGED=$(git diff --name-only origin/dev HEAD | grep "^autogpt_platform/backend/" && echo "true" || echo "false")
+        FRONTEND_CHANGED=$(git diff --name-only origin/dev HEAD | grep "^autogpt_platform/frontend/" && echo "true" || echo "false")
+        MARKET_CHANGED=$(git diff --name-only origin/dev HEAD | grep "^autogpt_platform/market/" && echo "true" || echo "false")
+        echo "backend_changed=$BACKEND_CHANGED" >> $GITHUB_OUTPUT
+        echo "frontend_changed=$FRONTEND_CHANGED" >> $GITHUB_OUTPUT
+        echo "market_changed=$MARKET_CHANGED" >> $GITHUB_OUTPUT
+
+    - name: Get GKE credentials
+      uses: 'google-github-actions/get-gke-credentials@v1'
+      with:
+        cluster_name: ${{ env.GKE_CLUSTER }}
+        location: ${{ env.GKE_ZONE }}
+
+    - name: Build and Push Backend
+      if: steps.check_changes.outputs.backend_changed == 'true'
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        file: ./autogpt_platform/backend/Dockerfile
+        push: true
+        tags: us-east1-docker.pkg.dev/agpt-dev/agpt-backend-dev/agpt-backend-dev:${{ github.sha }}
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+
+    - name: Build and Push Frontend
+      if: steps.check_changes.outputs.frontend_changed == 'true'
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        file: ./autogpt_platform/frontend/Dockerfile
+        push: true
+        tags: us-east1-docker.pkg.dev/agpt-dev/agpt-frontend-dev/agpt-frontend-dev:${{ github.sha }}
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+
+    - name: Build and Push Market
+      if: steps.check_changes.outputs.market_changed == 'true'
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        file: ./autogpt_platform/market/Dockerfile
+        push: true
+        tags: us-east1-docker.pkg.dev/agpt-dev/agpt-market-dev/agpt-market-dev:${{ github.sha }}
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+
+    - name: Move cache
+      run: |
+        rm -rf /tmp/.buildx-cache
+        mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+    - name: Set up Helm
+      uses: azure/setup-helm@v1
+      with:
+        version: v3.4.0
+
+    - name: Deploy Backend
+      if: steps.check_changes.outputs.backend_changed == 'true'
+      run: |
+        helm upgrade autogpt-server ./autogpt-server \
+          --namespace ${{ env.NAMESPACE }} \
+          -f autogpt-server/values.yaml \
+          -f autogpt-server/values.dev.yaml \
+          --set image.tag=${{ github.sha }}
+
+    - name: Deploy Websocket
+      if: steps.check_changes.outputs.backend_changed == 'true'
+      run: |
+        helm upgrade autogpt-websocket-server ./autogpt-websocket-server \
+          --namespace ${{ env.NAMESPACE }} \
+          -f autogpt-websocket-server/values.yaml \
+          -f autogpt-websocket-server/values.dev.yaml \
+          --set image.tag=${{ github.sha }}
+
+    - name: Deploy Market
+      if: steps.check_changes.outputs.market_changed == 'true'
+      run: |
+        helm upgrade autogpt-market ./autogpt-market \
+          --namespace ${{ env.NAMESPACE }} \
+          -f autogpt-market/values.yaml \
+          -f autogpt-market/values.dev.yaml \
+          --set image.tag=${{ github.sha }}
+
+    - name: Deploy Frontend
+      if: steps.check_changes.outputs.frontend_changed == 'true'
+      run: |
+        helm upgrade autogpt-builder ./autogpt-builder \
+          --namespace ${{ env.NAMESPACE }} \
+          -f autogpt-builder/values.yaml \
+          -f autogpt-builder/values.dev.yaml \
+          --set image.tag=${{ github.sha }}

autogpt_platform/infra/helm/autogpt-builder/values.dev.yaml

@@ -1,9 +1,9 @@
 # dev values, overwrite base values as needed.
 
 image:
-  repository: us-east1-docker.pkg.dev/agpt-dev/agpt-builder-dev/agpt-builder-dev
+  repository: us-east1-docker.pkg.dev/agpt-dev/agpt-frontend-dev/agpt-frontend-dev
   pullPolicy: Always
-  tag: "fe3d2a9"
+  tag: "latest"
 
 serviceAccount:
   annotations:

autogpt_platform/infra/helm/autogpt-server/values.dev.yaml

@@ -1,7 +1,7 @@
 # dev values, overwrite base values as needed.
 
 image:
-  repository: us-east1-docker.pkg.dev/agpt-dev/agpt-server-dev/agpt-server-dev
+  repository: us-east1-docker.pkg.dev/agpt-dev/agpt-backend-dev/agpt-backend-dev
   pullPolicy: Always
   tag: "latest"
 

autogpt_platform/infra/helm/autogpt-websocket-server/values.dev.yaml

@@ -1,7 +1,7 @@
 replicaCount: 1 # not scaling websocket server for now
 
 image:
-  repository: us-east1-docker.pkg.dev/agpt-dev/agpt-server-dev/agpt-server-dev
+  repository: us-east1-docker.pkg.dev/agpt-dev/agpt-backend-dev/agpt-backend-dev
   tag: latest
   pullPolicy: Always
 

autogpt_platform/infra/terraform/environments/dev.tfvars

@@ -28,6 +28,10 @@ service_accounts = {
    "dev-agpt-market-sa" = {
     display_name = "AutoGPT Dev Market Server Account"
     description  = "Service account for agpt dev market server"
+  },
+  "dev-github-actions-sa" = {
+    display_name = "GitHub Actions Dev Service Account"
+    description  = "Service account for GitHub Actions deployments to dev"
   }
 }
 
@@ -51,6 +55,11 @@ workload_identity_bindings = {
     service_account_name = "dev-agpt-market-sa"
     namespace            = "dev-agpt"
     ksa_name             = "dev-agpt-market-sa"
+  },
+  "dev-github-actions-workload-identity" = {
+    service_account_name = "dev-github-actions-sa"
+    namespace            = "dev-agpt"
+    ksa_name             = "dev-github-actions-sa"
   }
 }
 
@@ -59,7 +68,8 @@ role_bindings = {
     "serviceAccount:[email protected]",
     "serviceAccount:[email protected]",
     "serviceAccount:[email protected]",
-    "serviceAccount:[email protected]"
+    "serviceAccount:[email protected]",
+    "serviceAccount:[email protected]"
   ],
   "roles/cloudsql.client" = [
     "serviceAccount:[email protected]",
@@ -80,7 +90,8 @@ role_bindings = {
     "serviceAccount:[email protected]",
     "serviceAccount:[email protected]",
     "serviceAccount:[email protected]",
-    "serviceAccount:[email protected]"
+    "serviceAccount:[email protected]",
+    "serviceAccount:[email protected]"
   ]
   "roles/compute.networkUser" = [
     "serviceAccount:[email protected]",
@@ -93,6 +104,16 @@ role_bindings = {
     "serviceAccount:[email protected]",
     "serviceAccount:[email protected]",
     "serviceAccount:[email protected]"
+  ],
+  "roles/artifactregistry.writer" = [
+    "serviceAccount:[email protected]"
+  ],
+  "roles/container.viewer" = [
+    "serviceAccount:[email protected]"
+  ],
+  "roles/iam.serviceAccountTokenCreator" = [
+    "principalSet://iam.googleapis.com/projects/638488734936/locations/global/workloadIdentityPools/dev-pool/*",
+    "serviceAccount:[email protected]"
   ]
 }
 
@@ -101,4 +122,25 @@ services_ip_cidr_range = "10.2.0.0/20"
 
 public_bucket_names = ["website-artifacts"]
 standard_bucket_names = []
-bucket_admins = ["[email protected]", "[email protected]"]
+bucket_admins = ["[email protected]", "[email protected]"]
+
+workload_identity_pools = {
+  "dev-pool" = {
+    display_name = "Development Identity Pool"
+    providers = {
+      "github" = {
+        issuer_uri = "https://token.actions.githubusercontent.com"
+        attribute_mapping = {
+          "google.subject" = "assertion.sub"
+          "attribute.repository" = "assertion.repository"
+          "attribute.repository_owner" = "assertion.repository_owner"
+        }
+      }
+    }
+    service_accounts = {
+      "dev-github-actions-sa" = [
+        "Significant-Gravitas/AutoGPT"
+      ]
+    }
+  }
+}

autogpt_platform/infra/terraform/main.tf

@@ -61,6 +61,7 @@ module "iam" {
   service_accounts           = var.service_accounts
   workload_identity_bindings = var.workload_identity_bindings
   role_bindings              = var.role_bindings
+  workload_identity_pools    = var.workload_identity_pools
 }
 
 module "storage" {

autogpt_platform/infra/terraform/modules/iam/main.tf

@@ -23,4 +23,31 @@ resource "google_project_iam_binding" "role_bindings" {
   role     = each.key
 
   members = each.value
+}
+
+resource "google_iam_workload_identity_pool" "pools" {
+  for_each = var.workload_identity_pools
+  workload_identity_pool_id = each.key
+  display_name = each.value.display_name
+}
+
+resource "google_iam_workload_identity_pool_provider" "providers" {
+  for_each = merge([
+    for pool_id, pool in var.workload_identity_pools : {
+      for provider_id, provider in pool.providers :
+      "${pool_id}/${provider_id}" => merge(provider, {
+        pool_id = pool_id
+      })
+    }
+  ]...)
+
+  workload_identity_pool_id = split("/", each.key)[0]
+  workload_identity_pool_provider_id = split("/", each.key)[1]
+
+  attribute_mapping = each.value.attribute_mapping
+  oidc {
+    issuer_uri = each.value.issuer_uri
+    allowed_audiences = each.value.allowed_audiences
+  }
+  attribute_condition = "assertion.repository_owner==\"Significant-Gravitas\""
 }

autogpt_platform/infra/terraform/modules/iam/outputs.tf

@@ -1,4 +1,14 @@
 output "service_account_emails" {
   description = "The emails of the created service accounts"
   value       = { for k, v in google_service_account.service_accounts : k => v.email }
-}
+}
+
+output "workload_identity_pools" {
+  value = google_iam_workload_identity_pool.pools
+}
+
+output "workload_identity_providers" {
+  value = {
+    for k, v in google_iam_workload_identity_pool_provider.providers : k => v.name
+  }
+}

autogpt_platform/infra/terraform/modules/iam/variables.tf

@@ -26,4 +26,17 @@ variable "role_bindings" {
   description = "Map of roles to list of members"
   type        = map(list(string))
   default     = {}
+}
+
+variable "workload_identity_pools" {
+  type = map(object({
+    display_name = string
+    providers = map(object({
+      issuer_uri = string
+      attribute_mapping = map(string)
+      allowed_audiences = optional(list(string))
+    }))
+    service_accounts = map(list(string))  # Map of SA to list of allowed principals
+  }))
+  default = {}
 }

autogpt_platform/infra/terraform/variables.tf

@@ -130,3 +130,19 @@ variable "bucket_admins" {
   default     = ["[email protected]", "[email protected]"]
 }
 
+variable "workload_identity_pools" {
+  type = map(object({
+    display_name = string
+    providers = map(object({
+      issuer_uri = string
+      attribute_mapping = map(string)
+      allowed_audiences = optional(list(string))
+    }))
+    service_accounts = map(list(string))
+  }))
+  default = {}
+  description = "Configuration for workload identity pools and their providers"
+}
+
+
+