feat!: Rewrite ASM module (terraform-google-modules#1140)

* Remove previous ASM module and add initial implementation of new * Move from kubernetes_manifest to bash CPR creation * Add meshconfig.googleapis.com service * Add namespace to CPR creation script wait * Change example to use google_container_cluster * Improve CPR status wait duration * Use retries rather than sleeping to wait for CPR CRD existence * Enable servicemesh feature in module * Revert changes to examples * Fix enable_mdp to just enable CNI * Lint * minor fixes * Bump timeout on status wait * Don't create MeshConfig if unset * Fix ASM sample * Update README autogen * Minor fixes * Remove meshConfig from module * fix end to end tests * lint fixes * Remove enable_mdp, add enable_vpc_sc and fleet_id * move VPC-SC from labels to annotations * update README * use default node pool size * use wip for exmaple cluster creation * add feature enablement back to module * lint * remove feature enablement from module * remove depends_on * fix unspecified channel bug * minor fixes * add more testing * update docs * change required module version * fix cclb * change registry source to run in CI * update CROSS_CLUSTER_SERVICE_DISCOVERY to multicluster_mode * update README * fix test * fix wording * remove from README * iterate on comments Co-authored-by: kaariger <[email protected]>
WALTER-GROUP · Mar 3, 2022 · 13f9c06 · 13f9c06
1 parent 904e925
commit 13f9c06
Show file tree

Hide file tree

Showing 12 changed files with 211 additions and 547 deletions.
diff --git a/docs/upgrading_to_v20.0.md b/docs/upgrading_to_v20.0.md
@@ -0,0 +1,10 @@
+# Upgrading to v20.0
+
+The v20.0 release of *kubernetes-engine* is a backwards incompatible
+release for the Anthos Service Mesh (ASM) module.
+
+### ASM module rewrite
+
+The [ASM submodule](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/asm) has been rewritten to use the `ControlPlaneRevision` API to provision
+a managed control plane rather than using an installer script. Due to the drastic difference in implementation the module does not support an upgrade path
+from the previous version.
diff --git a/examples/simple_zonal_with_asm/hub.tf b/examples/simple_zonal_with_asm/hub.tf
@@ -14,11 +14,20 @@
  * limitations under the License.
  */
 
-module "hub" {
-  source                  = "../../modules/hub"
-  project_id              = var.project_id
-  location                = module.gke.location
-  cluster_name            = module.gke.name
-  cluster_endpoint        = module.gke.endpoint
-  gke_hub_membership_name = "gke-asm-membership"
+resource "google_gke_hub_membership" "cluster_membership" {
+  provider      = google-beta
+  project       = var.project_id
+  membership_id = "gke-asm-membership"
+  endpoint {
+    gke_cluster {
+      resource_link = "//container.googleapis.com/${module.gke.cluster_id}"
+    }
+  }
+}
+
+resource "google_gke_hub_feature" "mesh" {
+  name     = "servicemesh"
+  project  = var.project_id
+  location = "global"
+  provider = google-beta
 }
diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf
@@ -44,29 +44,23 @@ module "gke" {
   ip_range_services       = var.ip_range_services
   network_policy          = false
   cluster_resource_labels = { "mesh_id" : "proj-${data.google_project.project.number}" }
+  identity_namespace      = "${var.project_id}.svc.id.goog"
   node_pools = [
     {
       name         = "asm-node-pool"
       autoscaling  = false
       auto_upgrade = true
-      # ASM requires minimum 4 nodes and e2-standard-4
-      node_count   = 4
+      node_count   = 3
       machine_type = "e2-standard-4"
     },
   ]
 }
 
 module "asm" {
-  source                    = "../../modules/asm"
-  cluster_name              = module.gke.name
-  cluster_endpoint          = module.gke.endpoint
-  project_id                = var.project_id
-  location                  = module.gke.location
-  enable_cluster_roles      = true
-  enable_cluster_labels     = true
-  enable_gcp_apis           = true
-  enable_gcp_components     = true
-  enable_namespace_creation = true
-  options                   = ["envoy-access-log"]
-  outdir                    = "./${module.gke.name}-outdir"
+  source            = "../../modules/asm"
+  project_id        = var.project_id
+  cluster_name      = module.gke.name
+  cluster_location  = module.gke.location
+  multicluster_mode = "connected"
+  enable_cni        = true
 }
diff --git a/modules/asm/README.md b/modules/asm/README.md
@@ -2,87 +2,49 @@
 
 This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh/docs) (ASM) in a Kubernetes Engine (GKE) cluster.
 
-Specifically, this module automates installing the ASM Istio Operator on your cluster ([installing ASM](https://cloud.google.com/service-mesh/docs/install)).
-
 ## Usage
 
-There is a [full example](../../examples/simple_zonal_with_asm) provided. Detailed usage example is as follows:
+There are a few prerequisites to using this module that can be done either through Terraform or manually:
+
+1. Enable the `mesh.cloud.googleapis.com` service
+1. Enable the `servicemesh` feature on the cluster hub
+1. Register target cluster to the servicemesh-enabled hub
+
+There is a full example provided [here](../../examples/simple_zonal_with_asm). Detailed usage example is as follows:
 
 ```tf
 module "asm" {
-  source                = "terraform-google-modules/kubernetes-engine/google//modules/asm"
-
-  project_id            = "my-project-id"
-  cluster_name          = "my-cluster-name"
-  location              = module.gke.location
-  cluster_endpoint      = module.gke.endpoint
-  enable_all            = false
-  enable_cluster_roles  = true
-  enable_cluster_labels = false
-  enable_gcp_apis       = false
-  enable_gcp_iam_roles  = true
-  enable_gcp_components = true
-  enable_registration   = false
-  managed_control_plane = false
-  options               = ["envoy-access-log,egressgateways"]
-  custom_overlays       = ["./custom_ingress_gateway.yaml"]
-  skip_validation       = true
-  outdir                = "./${module.gke.name}-outdir-${var.asm_version}"
+  source            = "../../modules/asm"
+  project_id        = var.project_id
+  cluster_name      = module.gke.name
+  cluster_location  = module.gke.location
+  enable_cni        = true
 }
 ```
 
 To deploy this config:
 
 1. Run `terraform apply`
 
-## Requirements
-
-- Anthos Service Mesh on GCP no longer requires an active Anthos license. You can use Anthos Service Mesh as a standalone product on GCP (on GKE) or as part of your Anthos subscription for hybrid and multi-cloud architectures.
-- GKE cluster must have minimum four nodes.
-- Minimum machine type is `e2-standard-4`.
-- GKE cluster must be enrolled in a release channel. ASM does not support static version.
-- ASM on a private GKE cluster requires adding a firewall rule to open port 15017 if you want to use [automatic sidecar injection](https://cloud.google.com/service-mesh/docs/proxy-injection).
-- One ASM mesh per Google Cloud project is supported.
-
  <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| asm\_git\_tag | ASM git tag to deploy. This module supports versions `1.8`, `1.9` and `1.10`. You can get the exact `asm_git_tag` by running the command `install_asm --version`. The ASM git tab should be of the form `1.9.3-asm.2+config5`. You can also see all ASM git tags by running `curl https://storage.googleapis.com/csm-artifacts/asm/STABLE_VERSIONS`. You must provide the full and exact git tag. This variable is optional. Leaving it empty (default) will download the latest `install_asm` script for the version provided by the `asm_version` variable. | `string` | `""` | no |
-| asm\_version | ASM version to deploy. This module supports versions `1.8`, `1.9` and `1.10`. Available versions are documented in https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages | `string` | `"1.9"` | no |
-| ca | Sets CA option. Possible values are `meshca` or `citadel`. Additional documentation on Citadel is available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_citadel_as_the_ca. | `string` | `"meshca"` | no |
-| ca\_certs | Sets CA certificate file paths when `ca` is set to `citadel`. These values must be provided when using Citadel as CA. Additional documentation on Citadel is available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_citadel_as_the_ca. | `map(any)` | `{}` | no |
-| cluster\_endpoint | The GKE cluster endpoint. | `string` | n/a | yes |
+| channel | The channel to use for this ASM installation. | `string` | `""` | no |
+| cluster\_location | The cluster location for this ASM installation. | `string` | n/a | yes |
 | cluster\_name | The unique name to identify the cluster in ASM. | `string` | n/a | yes |
-| custom\_overlays | Comma separated list of custom\_overlay file paths. Works with in-cluster control plane only. Additional documentation available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_an_overlay_file | `list(any)` | `[]` | no |
-| enable\_all | Sets `--enable_all` option if true. | `bool` | `false` | no |
-| enable\_cluster\_labels | Sets `--enable_cluster_labels` option if true. | `bool` | `false` | no |
-| enable\_cluster\_roles | Sets `--enable_cluster_roles` option if true. | `bool` | `false` | no |
-| enable\_gcp\_apis | Sets `--enable_gcp_apis` option if true. | `bool` | `false` | no |
-| enable\_gcp\_components | Sets --enable\_gcp\_components option if true. Can be true or false. Available versions are documented in https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages | `bool` | `false` | no |
-| enable\_gcp\_iam\_roles | Grants IAM roles required for ASM if true. If enable\_gcp\_iam\_roles, one of impersonate\_service\_account, service\_account, or iam\_member must be set. | `bool` | `false` | no |
-| enable\_namespace\_creation | Sets `--enable_namespace_creation` option if true. | `bool` | `false` | no |
-| enable\_registration | Sets `--enable_registration` option if true. | `bool` | `false` | no |
-| gcloud\_sdk\_version | The gcloud sdk version to use. Minimum required version is 293.0.0 | `string` | `"296.0.1"` | no |
-| iam\_member | The GCP member email address to grant IAM roles to. If impersonate\_service\_account or service\_account is set, roles are granted to that SA. | `string` | `""` | no |
-| impersonate\_service\_account | An optional service account to impersonate for gcloud commands. If this service account is not specified, the module will use Application Default Credentials. | `string` | `""` | no |
-| key\_file | The GCP Service Account credentials file path used to deploy ASM. | `string` | `""` | no |
-| location | The location (zone or region) this cluster has been created in. | `string` | n/a | yes |
-| managed\_control\_plane | ASM managed control plane boolean. Determines whether to install ASM managed control plane. Installing ASM managed control plane does not install gateways. Documentation on how to install gateways with ASM MCP can be found at https://cloud.google.com/service-mesh/docs/managed-control-plane#install_istio_gateways_optional. | `bool` | `false` | no |
-| mode | ASM mode for deployment. Supported modes are `install` and `upgrade`. | `string` | `"install"` | no |
-| options | Comma separated list of options. Works with in-cluster control plane only. Supported options are documented in https://cloud.google.com/service-mesh/docs/enable-optional-features. | `list(any)` | `[]` | no |
-| outdir | Sets `--outdir` option. | `string` | `"none"` | no |
+| enable\_cni | Determines whether to enable CNI for this ASM installation. Required to use Managed Data Plane (MDP). | `bool` | `false` | no |
+| enable\_vpc\_sc | Determines whether to enable VPC-SC for this ASM installation. For more information read https://cloud.google.com/service-mesh/docs/managed/vpc-sc | `bool` | `false` | no |
+| fleet\_id | The fleet to use for this ASM installation. | `string` | `""` | no |
+| multicluster\_mode | [Preview] Determines whether remote secrets should be autogenerated across fleet cluster. | `string` | `"manual"` | no |
 | project\_id | The project in which the resource belongs. | `string` | n/a | yes |
-| revision\_name | Sets `--revision-name` option. | `string` | `"none"` | no |
-| service\_account | The GCP Service Account email address used to deploy ASM. | `string` | `""` | no |
-| service\_account\_key\_file | Path to service account key file to auth as for running `gcloud container clusters get-credentials`. | `string` | `""` | no |
-| skip\_validation | Sets `_CI_NO_VALIDATE` variable. Determines whether the script should perform validation checks for prerequisites such as IAM roles, Google APIs etc. | `bool` | `false` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| asm\_wait | An output to use when you want to depend on ASM finishing |
+| revision\_name | The name of the installed managed ASM revision. |
+| wait | An output to use when depending on the ASM installation finishing. |
 
- <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,95 +14,50 @@
  * limitations under the License.
  */
 
-data "google_project" "asm_project" {
-  project_id = var.project_id
-}
-
 locals {
-  options_string         = length(var.options) > 0 ? join(",", var.options) : "none"
-  custom_overlays_string = length(var.custom_overlays) > 0 ? join(",", var.custom_overlays) : "none"
-  asm_git_tag_string     = (var.asm_git_tag == "" ? "none" : var.asm_git_tag)
-  service_account_string = (var.service_account == "" ? "none" : var.service_account)
-  key_file_string        = (var.key_file == "" ? "none" : var.key_file)
-  ca_cert                = lookup(var.ca_certs, "ca_cert", "none")
-  ca_key                 = lookup(var.ca_certs, "ca_key", "none")
-  root_cert              = lookup(var.ca_certs, "root_cert", "none")
-  cert_chain             = lookup(var.ca_certs, "cert_chain", "none")
-  revision_name_string   = (var.revision_name == "" ? "none" : var.revision_name)
-  asm_minor_version      = tonumber(split(".", var.asm_version)[1])
-  # https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages/blob/1cf61b679cd369f42a0e735f8e201de1a6a6433b/scripts/asm-installer/install_asm#L1970
-  iam_roles = [
-    "roles/container.admin",
-    "roles/meshconfig.admin",
-    "roles/gkehub.admin",
-  ]
-  # https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages/blob/1cf61b679cd369f42a0e735f8e201de1a6a6433b/scripts/asm-installer/install_asm#L1958
-  mcp_iam_roles = [
-    "roles/serviceusage.serviceUsageConsumer",
-    "roles/container.admin",
-    "roles/monitoring.metricWriter",
-    "roles/logging.logWriter",
-    "roles/gkehub.viewer",
-    "roles/gkehub.gatewayAdmin",
-  ]
-  # if enable_gcp_iam_roles is set, grant IAM roles to first non null principal in the order below
-  asm_iam_member = var.enable_gcp_iam_roles ? coalesce(var.impersonate_service_account, var.service_account, var.iam_member) : ""
-  # compute any additonal resources that ASM provisioner should depend on
-  additional_depends_on = concat(var.enable_gcp_apis ? [module.asm-services[0].project_id] : [], local.asm_iam_member != "" ? [for k, v in google_project_iam_member.asm_iam : v.etag] : [])
-  # base command template for ASM installation
-  kubectl_create_command_base = "${path.module}/scripts/install_asm.sh ${var.project_id} ${var.cluster_name} ${var.location} ${var.asm_version} ${var.mode} ${var.managed_control_plane} ${var.skip_validation} ${local.options_string} ${local.custom_overlays_string} ${var.enable_all} ${var.enable_cluster_roles} ${var.enable_cluster_labels} ${var.enable_gcp_components} ${var.enable_registration} ${var.outdir} ${var.ca} ${local.ca_cert} ${local.ca_key} ${local.root_cert} ${local.cert_chain} ${local.service_account_string} ${local.key_file_string} ${local.asm_git_tag_string} ${local.revision_name_string}"
+  // GKE release channel is a list with max length 1 https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
+  gke_release_channel          = data.google_container_cluster.asm.release_channel != null ? data.google_container_cluster.asm.release_channel[0].channel : ""
+  gke_release_channel_filtered = lower(local.gke_release_channel) == "unspecified" ? "" : local.gke_release_channel
+  // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
+  channel       = lower(coalesce(var.channel, local.gke_release_channel_filtered, "regular"))
+  revision_name = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
+  // Fleet ID should default to project ID if unset
+  fleet_id = coalesce(var.fleet_id, var.project_id)
 }
 
-resource "google_project_iam_member" "asm_iam" {
-  for_each = toset(local.asm_iam_member != "" ? (var.managed_control_plane ? local.mcp_iam_roles : local.iam_roles) : [])
+data "google_container_cluster" "asm" {
   project  = var.project_id
-  role     = each.value
-  member   = "serviceAccount:${local.asm_iam_member}"
+  name     = var.cluster_name
+  location = var.cluster_location
 }
 
-module "asm-services" {
-  source  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version = "~> 11.3"
-
-  count = var.enable_gcp_apis ? 1 : 0
+resource "kubernetes_namespace" "system" {
+  metadata {
+    name = "istio-system"
+  }
+}
 
-  project_id                  = var.project_id
-  disable_services_on_destroy = false
-  disable_dependent_services  = false
+resource "kubernetes_config_map" "asm_options" {
+  metadata {
+    name      = "asm-options"
+    namespace = kubernetes_namespace.system.metadata[0].name
+  }
 
-  # https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages/blob/1cf61b679cd369f42a0e735f8e201de1a6a6433b/scripts/asm-installer/install_asm#L2005
-  activate_apis = [
-    "container.googleapis.com",
-    "monitoring.googleapis.com",
-    "logging.googleapis.com",
-    "cloudtrace.googleapis.com",
-    "meshtelemetry.googleapis.com",
-    "meshconfig.googleapis.com",
-    "meshca.googleapis.com",
-    "iamcredentials.googleapis.com",
-    "gkeconnect.googleapis.com",
-    "gkehub.googleapis.com",
-    "cloudresourcemanager.googleapis.com",
-    "stackdriver.googleapis.com",
-  ]
+  data = {
+    multicluster_mode = var.multicluster_mode
+  }
 }
 
-module "asm_install" {
+module "cpr" {
   source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
   version = "~> 3.1"
 
-  module_depends_on = concat([var.cluster_endpoint], local.additional_depends_on)
+  project_id       = var.project_id
+  cluster_name     = var.cluster_name
+  cluster_location = var.cluster_location
 
-  gcloud_sdk_version          = var.gcloud_sdk_version
-  upgrade                     = true
-  additional_components       = ["kubectl", "kpt", "beta"]
-  cluster_name                = var.cluster_name
-  cluster_location            = var.location
-  project_id                  = var.project_id
-  service_account_key_file    = var.service_account_key_file
-  impersonate_service_account = var.impersonate_service_account
+  kubectl_create_command  = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${var.enable_cni} ${var.enable_vpc_sc}"
+  kubectl_destroy_command = "${path.module}/scripts/destroy_cpr.sh ${local.revision_name}"
 
-  # enable_namespace_creation flag is only available starting 1.10
-  kubectl_create_command  = (local.asm_minor_version > 9 ? "${local.kubectl_create_command_base} ${var.enable_namespace_creation}" : local.kubectl_create_command_base)
-  kubectl_destroy_command = "${path.module}/scripts/destroy_asm.sh"
+  module_depends_on = [kubernetes_config_map.asm_options]
 }
diff --git a/modules/asm/outputs.tf b/modules/asm/outputs.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,12 @@
  * limitations under the License.
  */
 
-output "asm_wait" {
-  description = "An output to use when you want to depend on ASM finishing"
-  value       = module.asm_install.wait
+output "revision_name" {
+  value       = local.revision_name
+  description = "The name of the installed managed ASM revision."
+}
+
+output "wait" {
+  value       = module.cpr.wait
+  description = "An output to use when depending on the ASM installation finishing."
 }
diff --git a/modules/asm/scripts/.gitignore b/modules/asm/scripts/.gitignore