From 285dd05b63f5979002f4552bcf3aa6a5cc0ba6c0 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Thu, 20 Jan 2022 16:28:09 -0800
Subject: [PATCH 01/42] Remove previous ASM module and add initial
 implementation of new

---
 modules/asm/README.md              |  77 ++--------
 modules/asm/main.tf                | 143 +++++++++----------
 modules/asm/outputs.tf             |   8 +-
 modules/asm/scripts/.gitignore     |   1 -
 modules/asm/scripts/destroy_asm.sh |  20 ---
 modules/asm/scripts/install_asm.sh | 217 -----------------------------
 modules/asm/variables.tf           | 205 +++++----------------------
 7 files changed, 112 insertions(+), 559 deletions(-)
 delete mode 100644 modules/asm/scripts/.gitignore
 delete mode 100755 modules/asm/scripts/destroy_asm.sh
 delete mode 100755 modules/asm/scripts/install_asm.sh

diff --git a/modules/asm/README.md b/modules/asm/README.md
index 9c5f8d5fcb..e95e20fd91 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -2,87 +2,26 @@
 
 This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh/docs) (ASM) in a Kubernetes Engine (GKE) cluster.
 
-Specifically, this module automates installing the ASM Istio Operator on your cluster ([installing ASM](https://cloud.google.com/service-mesh/docs/install)).
-
 ## Usage
 
-There is a [full example](../../examples/simple_zonal_with_asm) provided. Detailed usage example is as follows:
-
-```tf
-module "asm" {
-  source                = "terraform-google-modules/kubernetes-engine/google//modules/asm"
-
-  project_id            = "my-project-id"
-  cluster_name          = "my-cluster-name"
-  location              = module.gke.location
-  cluster_endpoint      = module.gke.endpoint
-  enable_all            = false
-  enable_cluster_roles  = true
-  enable_cluster_labels = false
-  enable_gcp_apis       = false
-  enable_gcp_iam_roles  = true
-  enable_gcp_components = true
-  enable_registration   = false
-  managed_control_plane = false
-  options               = ["envoy-access-log,egressgateways"]
-  custom_overlays       = ["./custom_ingress_gateway.yaml"]
-  skip_validation       = true
-  outdir                = "./${module.gke.name}-outdir-${var.asm_version}"
-}
-```
-
-To deploy this config:
-
-1. Run `terraform apply`
-
-## Requirements
-
-- Anthos Service Mesh on GCP no longer requires an active Anthos license. You can use Anthos Service Mesh as a standalone product on GCP (on GKE) or as part of your Anthos subscription for hybrid and multi-cloud architectures.
-- GKE cluster must have minimum four nodes.
-- Minimum machine type is `e2-standard-4`.
-- GKE cluster must be enrolled in a release channel. ASM does not support static version.
-- ASM on a private GKE cluster requires adding a firewall rule to open port 15017 if you want to use [automatic sidecar injection](https://cloud.google.com/service-mesh/docs/proxy-injection).
-- One ASM mesh per Google Cloud project is supported.
-
  <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| asm\_git\_tag | ASM git tag to deploy. This module supports versions `1.8`, `1.9` and `1.10`. You can get the exact `asm_git_tag` by running the command `install_asm --version`. The ASM git tab should be of the form `1.9.3-asm.2+config5`. You can also see all ASM git tags by running `curl https://storage.googleapis.com/csm-artifacts/asm/STABLE_VERSIONS`. You must provide the full and exact git tag. This variable is optional. Leaving it empty (default) will download the latest `install_asm` script for the version provided by the `asm_version` variable. | `string` | `""` | no |
-| asm\_version | ASM version to deploy. This module supports versions `1.8`, `1.9` and `1.10`. Available versions are documented in https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages | `string` | `"1.9"` | no |
-| ca | Sets CA option. Possible values are `meshca` or `citadel`. Additional documentation on Citadel is available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_citadel_as_the_ca. | `string` | `"meshca"` | no |
-| ca\_certs | Sets CA certificate file paths when `ca` is set to `citadel`. These values must be provided when using Citadel as CA. Additional documentation on Citadel is available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_citadel_as_the_ca. | `map(any)` | `{}` | no |
-| cluster\_endpoint | The GKE cluster endpoint. | `string` | n/a | yes |
+| channel | The channel to use for this ASM installation. | `string` | `""` | no |
+| cluster\_location | The cluster location for this ASM installation. | `string` | n/a | yes |
 | cluster\_name | The unique name to identify the cluster in ASM. | `string` | n/a | yes |
-| custom\_overlays | Comma separated list of custom\_overlay file paths. Works with in-cluster control plane only. Additional documentation available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_an_overlay_file | `list(any)` | `[]` | no |
-| enable\_all | Sets `--enable_all` option if true. | `bool` | `false` | no |
-| enable\_cluster\_labels | Sets `--enable_cluster_labels` option if true. | `bool` | `false` | no |
-| enable\_cluster\_roles | Sets `--enable_cluster_roles` option if true. | `bool` | `false` | no |
-| enable\_gcp\_apis | Sets `--enable_gcp_apis` option if true. | `bool` | `false` | no |
-| enable\_gcp\_components | Sets --enable\_gcp\_components option if true. Can be true or false. Available versions are documented in https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages | `bool` | `false` | no |
-| enable\_gcp\_iam\_roles | Grants IAM roles required for ASM if true. If enable\_gcp\_iam\_roles, one of impersonate\_service\_account, service\_account, or iam\_member must be set. | `bool` | `false` | no |
-| enable\_namespace\_creation | Sets `--enable_namespace_creation` option if true. | `bool` | `false` | no |
-| enable\_registration | Sets `--enable_registration` option if true. | `bool` | `false` | no |
-| gcloud\_sdk\_version | The gcloud sdk version to use. Minimum required version is 293.0.0 | `string` | `"296.0.1"` | no |
-| iam\_member | The GCP member email address to grant IAM roles to. If impersonate\_service\_account or service\_account is set, roles are granted to that SA. | `string` | `""` | no |
-| impersonate\_service\_account | An optional service account to impersonate for gcloud commands. If this service account is not specified, the module will use Application Default Credentials. | `string` | `""` | no |
-| key\_file | The GCP Service Account credentials file path used to deploy ASM. | `string` | `""` | no |
-| location | The location (zone or region) this cluster has been created in. | `string` | n/a | yes |
-| managed\_control\_plane | ASM managed control plane boolean. Determines whether to install ASM managed control plane. Installing ASM managed control plane does not install gateways. Documentation on how to install gateways with ASM MCP can be found at https://cloud.google.com/service-mesh/docs/managed-control-plane#install_istio_gateways_optional. | `bool` | `false` | no |
-| mode | ASM mode for deployment. Supported modes are `install` and `upgrade`. | `string` | `"install"` | no |
-| options | Comma separated list of options. Works with in-cluster control plane only. Supported options are documented in https://cloud.google.com/service-mesh/docs/enable-optional-features. | `list(any)` | `[]` | no |
-| outdir | Sets `--outdir` option. | `string` | `"none"` | no |
+| enable\_cni | Determines whether to enable CNI for this ASM installation. | `bool` | `true` | no |
+| enable\_cross\_cluster\_service\_discovery | Determines whether to enable cross-cluster service discovery between this cluster and other clusters in the fleet. | `bool` | `false` | no |
+| enable\_mdp | Determines whether to enable Managed Data Plane (MDP) for this ASM installation. | `bool` | `true` | no |
+| mesh\_config | MeshConfig specifies configuration available to the control plane. The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig | `map` | `{}` | no |
 | project\_id | The project in which the resource belongs. | `string` | n/a | yes |
-| revision\_name | Sets `--revision-name` option. | `string` | `"none"` | no |
-| service\_account | The GCP Service Account email address used to deploy ASM. | `string` | `""` | no |
-| service\_account\_key\_file | Path to service account key file to auth as for running `gcloud container clusters get-credentials`. | `string` | `""` | no |
-| skip\_validation | Sets `_CI_NO_VALIDATE` variable. Determines whether the script should perform validation checks for prerequisites such as IAM roles, Google APIs etc. | `bool` | `false` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| asm\_wait | An output to use when you want to depend on ASM finishing |
+| revision\_name | The name of the installed managed ASM revision. |
 
- <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index 84e3828c39..0049fe0a9e 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,95 +14,78 @@
  * limitations under the License.
  */
 
-data "google_project" "asm_project" {
-  project_id = var.project_id
-}
-
 locals {
-  options_string         = length(var.options) > 0 ? join(",", var.options) : "none"
-  custom_overlays_string = length(var.custom_overlays) > 0 ? join(",", var.custom_overlays) : "none"
-  asm_git_tag_string     = (var.asm_git_tag == "" ? "none" : var.asm_git_tag)
-  service_account_string = (var.service_account == "" ? "none" : var.service_account)
-  key_file_string        = (var.key_file == "" ? "none" : var.key_file)
-  ca_cert                = lookup(var.ca_certs, "ca_cert", "none")
-  ca_key                 = lookup(var.ca_certs, "ca_key", "none")
-  root_cert              = lookup(var.ca_certs, "root_cert", "none")
-  cert_chain             = lookup(var.ca_certs, "cert_chain", "none")
-  revision_name_string   = (var.revision_name == "" ? "none" : var.revision_name)
-  asm_minor_version      = tonumber(split(".", var.asm_version)[1])
-  # https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages/blob/1cf61b679cd369f42a0e735f8e201de1a6a6433b/scripts/asm-installer/install_asm#L1970
-  iam_roles = [
-    "roles/container.admin",
-    "roles/meshconfig.admin",
-    "roles/gkehub.admin",
-  ]
-  # https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages/blob/1cf61b679cd369f42a0e735f8e201de1a6a6433b/scripts/asm-installer/install_asm#L1958
-  mcp_iam_roles = [
-    "roles/serviceusage.serviceUsageConsumer",
-    "roles/container.admin",
-    "roles/monitoring.metricWriter",
-    "roles/logging.logWriter",
-    "roles/gkehub.viewer",
-    "roles/gkehub.gatewayAdmin",
-  ]
-  # if enable_gcp_iam_roles is set, grant IAM roles to first non null principal in the order below
-  asm_iam_member = var.enable_gcp_iam_roles ? coalesce(var.impersonate_service_account, var.service_account, var.iam_member) : ""
-  # compute any additonal resources that ASM provisioner should depend on
-  additional_depends_on = concat(var.enable_gcp_apis ? [module.asm-services[0].project_id] : [], local.asm_iam_member != "" ? [for k, v in google_project_iam_member.asm_iam : v.etag] : [])
-  # base command template for ASM installation
-  kubectl_create_command_base = "${path.module}/scripts/install_asm.sh ${var.project_id} ${var.cluster_name} ${var.location} ${var.asm_version} ${var.mode} ${var.managed_control_plane} ${var.skip_validation} ${local.options_string} ${local.custom_overlays_string} ${var.enable_all} ${var.enable_cluster_roles} ${var.enable_cluster_labels} ${var.enable_gcp_components} ${var.enable_registration} ${var.outdir} ${var.ca} ${local.ca_cert} ${local.ca_key} ${local.root_cert} ${local.cert_chain} ${local.service_account_string} ${local.key_file_string} ${local.asm_git_tag_string} ${local.revision_name_string}"
+  // GKE release channel is a list with max length 1 for some reason https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
+  gke_release_channel = length(data.google_container_cluster.asm_cluster.release_channel) > 0 ? data.google_container_cluster.asm_cluster.release_channel[0].channel : ""
+  gke_release_channel_fixed = local.gke_release_channel == "UNSPECIFIED" ? "" : local.gke_release_channel
+  // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
+  channel = lower(coalesce(var.channel, local.gke_release_channel_fixed, "regular"))
+  revision_name = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
+  mesh_config_name= "istio-${local.revision_name}"
 }
 
-resource "google_project_iam_member" "asm_iam" {
-  for_each = toset(local.asm_iam_member != "" ? (var.managed_control_plane ? local.mcp_iam_roles : local.iam_roles) : [])
-  project  = var.project_id
-  role     = each.value
-  member   = "serviceAccount:${local.asm_iam_member}"
+data "google_container_cluster" "asm_cluster" {
+  project = var.project_id
+  name = var.cluster_name
+  location = var.cluster_location
 }
 
-module "asm-services" {
-  source  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version = "~> 11.3"
+resource "kubernetes_manifest" "cpr" {
+  manifest = {
+    "apiVersion" = "mesh.cloud.google.com/v1beta1"
+    "kind" = "ControlPlaneRevision"
+    "metadata" = {
+      "name" = local.revision_name
+      "namespace" = "istio-system"
+      "labels" = {
+        "mesh.cloud.google.com/managed-cni-enabled" = var.enable_cni
+      }
+    }
+    spec = {
+      type = "managed_service"
+      channel = local.channel
+    }
+  }
 
-  count = var.enable_gcp_apis ? 1 : 0
+  wait_for = {
+    fields = {
+      // In a perfect world we could do wait_for { condition = "Provisioned" } as described in https://github.com/hashicorp/terraform-provider-kubernetes-alpha/issues/12
+      // but it isn't implemented.
+      "status.conditions[0].reason": "Provisioned",
+      "status.conditions[0].status": "True"
+    }
+  }
 
-  project_id                  = var.project_id
-  disable_services_on_destroy = false
-  disable_dependent_services  = false
+  timeouts {
+    create = "10m"
+  }
 
-  # https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages/blob/1cf61b679cd369f42a0e735f8e201de1a6a6433b/scripts/asm-installer/install_asm#L2005
-  activate_apis = [
-    "container.googleapis.com",
-    "monitoring.googleapis.com",
-    "logging.googleapis.com",
-    "cloudtrace.googleapis.com",
-    "meshtelemetry.googleapis.com",
-    "meshconfig.googleapis.com",
-    "meshca.googleapis.com",
-    "iamcredentials.googleapis.com",
-    "gkeconnect.googleapis.com",
-    "gkehub.googleapis.com",
-    "cloudresourcemanager.googleapis.com",
-    "stackdriver.googleapis.com",
-  ]
+  depends_on = [kubernetes_config_map.asm_options, kubernetes_config_map.mesh_config]
 }
 
-module "asm_install" {
-  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
-  version = "~> 3.1"
-
-  module_depends_on = concat([var.cluster_endpoint], local.additional_depends_on)
+resource "kubernetes_config_map" "mesh_config" {
+    metadata {
+      name = local.mesh_config_name
+      namespace = "istio-system"
+      annotations = {
+        "mesh.cloud.google.com/proxy" = "{\"managed\": \"${var.enable_mdp}\"}"
+      }
+      labels = {
+        "istio.io/rev" = local.revision_name
+      }
+    }
+    data = {
+      mesh = yamlencode(var.mesh_config)
+    }
+}
 
-  gcloud_sdk_version          = var.gcloud_sdk_version
-  upgrade                     = true
-  additional_components       = ["kubectl", "kpt", "beta"]
-  cluster_name                = var.cluster_name
-  cluster_location            = var.location
-  project_id                  = var.project_id
-  service_account_key_file    = var.service_account_key_file
-  impersonate_service_account = var.impersonate_service_account
+resource "kubernetes_config_map" "asm_options" {
+  metadata {
+    name = "asm-options"
+    namespace = "istio-system"
+  }
 
-  # enable_namespace_creation flag is only available starting 1.10
-  kubectl_create_command  = (local.asm_minor_version > 9 ? "${local.kubectl_create_command_base} ${var.enable_namespace_creation}" : local.kubectl_create_command_base)
-  kubectl_destroy_command = "${path.module}/scripts/destroy_asm.sh"
+  data = {
+    CROSS_CLUSTER_SERVICE_DISCOVERY = var.enable_cross_cluster_service_discovery ? "ON" : "OFF"
+  }
 }
diff --git a/modules/asm/outputs.tf b/modules/asm/outputs.tf
index 7b806b515d..969ff8699c 100644
--- a/modules/asm/outputs.tf
+++ b/modules/asm/outputs.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-output "asm_wait" {
-  description = "An output to use when you want to depend on ASM finishing"
-  value       = module.asm_install.wait
+output "revision_name" {
+  value = local.revision_name
+  description = "The name of the installed managed ASM revision."
 }
diff --git a/modules/asm/scripts/.gitignore b/modules/asm/scripts/.gitignore
deleted file mode 100644
index a0e7b1004b..0000000000
--- a/modules/asm/scripts/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-install_asm
diff --git a/modules/asm/scripts/destroy_asm.sh b/modules/asm/scripts/destroy_asm.sh
deleted file mode 100755
index 46f91bde4a..0000000000
--- a/modules/asm/scripts/destroy_asm.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-kubectl delete ns asm-system istio-system --ignore-not-found
-kubectl label namespaces --all istio-injection-
diff --git a/modules/asm/scripts/install_asm.sh b/modules/asm/scripts/install_asm.sh
deleted file mode 100755
index 3be6f6d92f..0000000000
--- a/modules/asm/scripts/install_asm.sh
+++ /dev/null
@@ -1,217 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-if [ "$#" -lt 5 ]; then
-    >&2 echo "Not all expected arguments set."
-    exit 1
-fi
-
-PROJECT_ID=${1}
-CLUSTER_NAME=${2}
-CLUSTER_LOCATION=${3}
-ASM_VERSION=${4}
-MODE=${5}
-MCP=${6}
-SKIP_VALIDATION=${7}
-OPTIONS_LIST=${8}
-CUSTOM_OVERLAYS_LIST=${9}
-ENABLE_ALL=${10}
-ENABLE_CLUSTER_ROLES=${11}
-ENABLE_CLUSTER_LABELS=${12}
-ENABLE_GCP_COMPONENTS=${13}
-ENABLE_REGISTRATION=${14}
-OUTDIR=${15}
-CA=${16}
-CA_CERT=${17}
-CA_KEY=${18}
-ROOT_CERT=${19}
-CERT_CHAIN=${20}
-SERVICE_ACCOUNT=${21}
-KEY_FILE=${22}
-ASM_GIT_TAG=${23}
-REVISION_NAME=${24}
-ENABLE_NAMESPACE_CREATION=${25}
-
-# Set SKIP_VALIDATION variable
-if [[ ${SKIP_VALIDATION} = "true" ]]; then
-    export _CI_NO_VALIDATE=1
-else
-    export _CI_NO_VALIDATE=0
-fi
-
-# Create bash arrays from options and custom_overlays lists
-if [[ ${OPTIONS_LIST} ]]; then
-    IFS=',' read -r -a OPTIONS <<< "${OPTIONS_LIST}"
-elif [[ ${OPTIONS_LIST} = "" ]]; then
-    read -r -a OPTIONS <<< "none"
-fi
-
-if [[ ${CUSTOM_OVERLAYS_LIST} ]]; then
-    IFS=',' read -r -a CUSTOM_OVERLAYS <<< "${CUSTOM_OVERLAYS_LIST}"
-else
-    read -r -a CUSTOM_OVERLAYS <<< "none"
-fi
-
-# Echo all values
-echo -e "MODE is $MODE"
-echo -e "MCP is $MCP"
-echo -e "ASM_VERSION is $ASM_VERSION"
-echo -e "ASM_GIT_TAG is $ASM_GIT_TAG"
-echo -e "SKIP_VALIDATION is $SKIP_VALIDATION"
-echo -e "_CI_NO_VALIDATE is $_CI_NO_VALIDATE"
-echo -e "OPTIONS_LIST is ${OPTIONS_LIST}"
-echo -e "OPTIONS array length is ${#OPTIONS[@]}"
-# Create options command snippet
-item="${OPTIONS[*]}";OPTIONS_COMMAND=$(echo "--option" "${item// / --option }")
-echo -e "OPTIONS_COMMAND is $OPTIONS_COMMAND"
-echo -e "CUSTOM_OVERLAYS array length is ${#CUSTOM_OVERLAYS[@]}"
-# Create custom_overlays command snippet
-if [[ "${CUSTOM_OVERLAYS[*]}" == "none" ]]; then
-    CUSTOM_OVERLAYS_COMMAND="--custom_overlay none"
-else
-    item="${CUSTOM_OVERLAYS[*]}";CUSTOM_OVERLAYS_COMMAND=$(echo "--custom_overlay" "$(pwd)/${item// / --custom_overlay $(pwd)/}")
-fi
-echo -e "CUSTOM_OVERLAYS_COMMAND is $CUSTOM_OVERLAYS_COMMAND"
-echo -e "ENABLE_ALL is $ENABLE_ALL"
-echo -e "ENABLE_CLUSTER_ROLES is $ENABLE_CLUSTER_ROLES"
-echo -e "ENABLE_CLUSTER_LABELS is $ENABLE_CLUSTER_LABELS"
-echo -e "ENABLE_GCP_COMPONENTS is $ENABLE_GCP_COMPONENTS"
-echo -e "ENABLE_REGISTRATION is $ENABLE_REGISTRATION"
-echo -e "ENABLE_NAMESPACE_CREATION is $ENABLE_NAMESPACE_CREATION"
-echo -e "OUTDIR is $OUTDIR"
-echo -e "SERVICE_ACCOUNT is $SERVICE_ACCOUNT"
-echo -e "KEY_FILE is $KEY_FILE"
-echo -e "REVISION_NAME is $REVISION_NAME"
-echo -e "CA is $CA"
-echo -e "CA_CERT is $CA_CERT"
-echo -e "CA_KEY is $CA_KEY"
-echo -e "ROOT_CERT is $ROOT_CERT"
-echo -e "CERT_CHAIN is $CERT_CHAIN"
-#download the correct version of the install_asm script
-if [[ "${ASM_GIT_TAG}" = "none" ]]; then
-    echo -e "Downloading install_asm with latest git tag..."
-    curl https://storage.googleapis.com/csm-artifacts/asm/install_asm_"${ASM_VERSION}" > install_asm_"${ASM_VERSION}"
-    chmod u+x install_asm_"${ASM_VERSION}"
-else
-    ASM_GIT_TAG_FIXED=$(sed 's/+/-/g' <<<"$ASM_GIT_TAG")
-    echo -e "Downloading install_asm with git tag $ASM_GIT_TAG..."
-    curl https://storage.googleapis.com/csm-artifacts/asm/install_asm_"${ASM_GIT_TAG_FIXED}" > install_asm_"${ASM_VERSION}"
-    chmod u+x install_asm_"${ASM_VERSION}"
-fi
-
-# Craft MCP section for install_asm
-if [[ "${MCP}" = true ]]; then
-    MCP_COMMAND_SNIPPET="--managed"
-else
-    MCP_COMMAND_SNIPPET=""
-fi
-
-# Craft service_account section for install_asm
-if [[ "${SERVICE_ACCOUNT}" = "none" ]]; then
-    SERVICE_ACCOUNT_COMMAND_SNIPPET=""
-else
-    SERVICE_ACCOUNT_COMMAND_SNIPPET="--service_account ${SERVICE_ACCOUNT}"
-fi
-
-# Craft key_file section for install_asm
-if [[ "${KEY_FILE}" = "none" ]]; then
-    KEY_FILE_COMMAND_SNIPPET=""
-else
-    KEY_FILE_COMMAND_SNIPPET="--key_file $(pwd)/${KEY_FILE}"
-fi
-
-# Craft options section for install_asm
-if [[ "${OPTIONS_COMMAND}" = "--option none" ]]; then
-    OPTIONS_COMMAND_SNIPPET=""
-else
-    OPTIONS_COMMAND_SNIPPET="${OPTIONS_COMMAND}"
-fi
-
-if [[ "${CUSTOM_OVERLAYS_COMMAND}" = "--custom_overlay none" ]]; then
-    CUSTOM_OVERLAYS_COMMAND_SNIPPET=""
-else
-    CUSTOM_OVERLAYS_COMMAND_SNIPPET="${CUSTOM_OVERLAYS_COMMAND}"
-fi
-
-if [[ "${ENABLE_ALL}" = false ]]; then
-    ENABLE_ALL_COMMAND_SNIPPET=""
-else
-    ENABLE_ALL_COMMAND_SNIPPET="--enable_all"
-fi
-
-if [[ "${ENABLE_CLUSTER_ROLES}" = false ]]; then
-    ENABLE_CLUSTER_ROLES_COMMAND_SNIPPET=""
-else
-    ENABLE_CLUSTER_ROLES_COMMAND_SNIPPET="--enable_cluster_roles"
-fi
-
-if [[ "${ENABLE_CLUSTER_LABELS}" = false ]]; then
-    ENABLE_CLUSTER_LABELS_COMMAND_SNIPPET=""
-else
-    ENABLE_CLUSTER_LABELS_COMMAND_SNIPPET="--enable_cluster_labels"
-fi
-
-if [[ "${ENABLE_GCP_COMPONENTS}" = false ]]; then
-    ENABLE_GCP_COMPONENTS_COMMAND_SNIPPET=""
-else
-    ENABLE_GCP_COMPONENTS_COMMAND_SNIPPET="--enable_gcp_components"
-fi
-
-if [[ "${ENABLE_REGISTRATION}" = false ]]; then
-    ENABLE_REGISTRATION_COMMAND_SNIPPET=""
-else
-    ENABLE_REGISTRATION_COMMAND_SNIPPET="--enable_registration"
-fi
-
-if [[ "${ENABLE_NAMESPACE_CREATION}" = true ]]; then
-    ENABLE_NAMESPACE_CREATION_COMMAND_SNIPPET="--enable_namespace_creation"
-else
-    ENABLE_NAMESPACE_CREATION_COMMAND_SNIPPET=""
-fi
-
-if [[ "${OUTDIR}" = "none" ]]; then
-    OUTDIR_COMMAND_SNIPPET=""
-else
-    OUTDIR_COMMAND_SNIPPET="--output_dir ${OUTDIR}"
-    mkdir -p "${OUTDIR}"
-fi
-
-if [[ "${CA}" == "citadel" ]]; then
-    CA_COMMAND_SNIPPET="--ca citadel"
-else
-    CA_COMMAND_SNIPPET=""
-fi
-
-if [[ "${CA_CERT}" == "none" ]]; then
-    CA_CERTS_COMMAND_SNIPPET=""
-else
-    CA_CERTS_COMMAND_SNIPPET="--ca_cert ${CA_CERT} --ca_key ${CA_KEY} --root_cert ${ROOT_CERT} --cert_chain ${CERT_CHAIN}"
-fi
-
-if [[ "${REVISION_NAME}" == "none" ]]; then
-    REVISION_NAME_COMMAND_SNIPPET=""
-else
-    REVISION_NAME_COMMAND_SNIPPET="--revision_name ${REVISION_NAME}"
-fi
-
-# Echo the command before executing
-echo -e "install_asm_${ASM_VERSION} --verbose --project_id ${PROJECT_ID} --cluster_name ${CLUSTER_NAME} --cluster_location ${CLUSTER_LOCATION} --mode ${MODE} ${MCP_COMMAND_SNIPPET} ${OPTIONS_COMMAND_SNIPPET} ${CUSTOM_OVERLAYS_COMMAND_SNIPPET} ${OUTDIR_COMMAND_SNIPPET} ${ENABLE_ALL_COMMAND_SNIPPET} ${ENABLE_CLUSTER_ROLES_COMMAND_SNIPPET} ${ENABLE_CLUSTER_LABELS_COMMAND_SNIPPET} ${ENABLE_GCP_COMPONENTS_COMMAND_SNIPPET} ${ENABLE_REGISTRATION_COMMAND_SNIPPET} ${ENABLE_NAMESPACE_CREATION_COMMAND_SNIPPET} ${CA_COMMAND_SNIPPET} ${CA_CERTS_COMMAND_SNIPPET} ${SERVICE_ACCOUNT_COMMAND_SNIPPET} ${KEY_FILE_COMMAND_SNIPPET} ${REVISION_NAME_COMMAND_SNIPPET}"
-
-# run the script with appropriate flags
-# shellcheck disable=SC2086
-./install_asm_${ASM_VERSION} --verbose --project_id ${PROJECT_ID} --cluster_name ${CLUSTER_NAME} --cluster_location ${CLUSTER_LOCATION} --mode ${MODE} ${MCP_COMMAND_SNIPPET} ${OPTIONS_COMMAND_SNIPPET} ${CUSTOM_OVERLAYS_COMMAND_SNIPPET} ${OUTDIR_COMMAND_SNIPPET} ${ENABLE_ALL_COMMAND_SNIPPET} ${ENABLE_CLUSTER_ROLES_COMMAND_SNIPPET} ${ENABLE_CLUSTER_LABELS_COMMAND_SNIPPET} ${ENABLE_GCP_COMPONENTS_COMMAND_SNIPPET} ${ENABLE_REGISTRATION_COMMAND_SNIPPET} ${ENABLE_NAMESPACE_CREATION_COMMAND_SNIPPET} ${CA_COMMAND_SNIPPET} ${CA_CERTS_COMMAND_SNIPPET} ${SERVICE_ACCOUNT_COMMAND_SNIPPET} ${KEY_FILE_COMMAND_SNIPPET}  ${REVISION_NAME_COMMAND_SNIPPET}
diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf
index a3002eda3d..6aab28a5ee 100644
--- a/modules/asm/variables.tf
+++ b/modules/asm/variables.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,187 +14,56 @@
  * limitations under the License.
  */
 
-variable "cluster_name" {
-  description = "The unique name to identify the cluster in ASM."
-  type        = string
-}
-
-variable "cluster_endpoint" {
-  description = "The GKE cluster endpoint."
-  type        = string
-}
-
 variable "project_id" {
   description = "The project in which the resource belongs."
-  type        = string
-}
-
-variable "location" {
-  description = "The location (zone or region) this cluster has been created in."
-  type        = string
-}
-
-variable "gcloud_sdk_version" {
-  description = "The gcloud sdk version to use. Minimum required version is 293.0.0"
-  type        = string
-  default     = "296.0.1"
-}
-
-# variable "asm_dir" {
-#   description = "Name of directory to keep ASM resource config files."
-#   type        = string
-#   default     = "asm-dir"
-# }
-
-variable "service_account_key_file" {
-  description = "Path to service account key file to auth as for running `gcloud container clusters get-credentials`."
-  default     = ""
+  type = string
 }
 
-variable "asm_version" {
-  description = "ASM version to deploy. This module supports versions `1.8`, `1.9` and `1.10`. Available versions are documented in https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages"
-  type        = string
-  default     = "1.9"
-}
-
-variable "asm_git_tag" {
-  description = "ASM git tag to deploy. This module supports versions `1.8`, `1.9` and `1.10`. You can get the exact `asm_git_tag` by running the command `install_asm --version`. The ASM git tab should be of the form `1.9.3-asm.2+config5`. You can also see all ASM git tags by running `curl https://storage.googleapis.com/csm-artifacts/asm/STABLE_VERSIONS`. You must provide the full and exact git tag. This variable is optional. Leaving it empty (default) will download the latest `install_asm` script for the version provided by the `asm_version` variable."
-  type        = string
-  default     = ""
-}
-
-variable "mode" {
-  description = "ASM mode for deployment. Supported modes are `install` and `upgrade`."
-  type        = string
-  default     = "install"
-}
-
-variable "service_account" {
-  description = "The GCP Service Account email address used to deploy ASM."
-  type        = string
-  default     = ""
-}
-
-variable "key_file" {
-  description = "The GCP Service Account credentials file path used to deploy ASM."
-  type        = string
-  default     = ""
-}
-
-variable "managed_control_plane" {
-  description = "ASM managed control plane boolean. Determines whether to install ASM managed control plane. Installing ASM managed control plane does not install gateways. Documentation on how to install gateways with ASM MCP can be found at https://cloud.google.com/service-mesh/docs/managed-control-plane#install_istio_gateways_optional."
-  type        = bool
-  default     = false
-}
-
-variable "impersonate_service_account" {
-  type        = string
-  description = "An optional service account to impersonate for gcloud commands. If this service account is not specified, the module will use Application Default Credentials."
-  default     = ""
-}
-
-variable "options" {
-  description = "Comma separated list of options. Works with in-cluster control plane only. Supported options are documented in https://cloud.google.com/service-mesh/docs/enable-optional-features."
-  type        = list(any)
-  default     = []
-}
-
-variable "custom_overlays" {
-  description = "Comma separated list of custom_overlay file paths. Works with in-cluster control plane only. Additional documentation available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_an_overlay_file"
-  type        = list(any)
-  default     = []
-}
-
-variable "skip_validation" {
-  description = "Sets `_CI_NO_VALIDATE` variable. Determines whether the script should perform validation checks for prerequisites such as IAM roles, Google APIs etc."
-  type        = bool
-  default     = false
-}
-
-variable "enable_all" {
-  description = "Sets `--enable_all` option if true."
-  type        = bool
-  default     = false
-}
-
-variable "enable_cluster_roles" {
-  description = "Sets `--enable_cluster_roles` option if true."
-  type        = bool
-  default     = false
-}
-
-variable "enable_cluster_labels" {
-  description = "Sets `--enable_cluster_labels` option if true."
-  type        = bool
-  default     = false
-}
-
-variable "enable_gcp_apis" {
-  description = "Sets `--enable_gcp_apis` option if true."
-  type        = bool
-  default     = false
-}
-
-variable "enable_gcp_iam_roles" {
-  description = "Grants IAM roles required for ASM if true. If enable_gcp_iam_roles, one of impersonate_service_account, service_account, or iam_member must be set."
-  type        = bool
-  default     = false
-}
-
-variable "enable_gcp_components" {
-  description = "Sets --enable_gcp_components option if true. Can be true or false. Available versions are documented in https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages"
-  type        = bool
-  default     = false
-}
-
-variable "enable_registration" {
-  description = "Sets `--enable_registration` option if true."
-  type        = bool
-  default     = false
+variable "cluster_name" {
+  description = "The unique name to identify the cluster in ASM."
+  type = string
 }
 
-variable "enable_namespace_creation" {
-  description = "Sets `--enable_namespace_creation` option if true."
-  type        = bool
-  default     = false
+variable "cluster_location" {
+  description = "The cluster location for this ASM installation."
+  type = string
 }
 
-variable "outdir" {
-  description = "Sets `--outdir` option."
-  type        = string
-  default     = "none"
+variable "channel" {
+  description = "The channel to use for this ASM installation."
+  type = string
+  validation {
+    condition = anytrue([
+      var.channel == "rapid",
+      var.channel == "regular",
+      var.channel == "stable",
+      var.channel == "", // if unset, use GKE data source and use release cluster channel
+    ])
+    error_message = "Must be one of rapid, regular, or stable."
+  }
+  default = ""
 }
 
-variable "ca" {
-  description = "Sets CA option. Possible values are `meshca` or `citadel`. Additional documentation on Citadel is available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_citadel_as_the_ca."
-  type        = string
-  default     = "meshca"
+variable "enable_cni" {
+  description = "Determines whether to enable CNI for this ASM installation."
+  type = bool
+  default = true
 }
 
-variable "ca_certs" {
-  description = "Sets CA certificate file paths when `ca` is set to `citadel`. These values must be provided when using Citadel as CA. Additional documentation on Citadel is available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_citadel_as_the_ca."
-  type        = map(any)
-  default     = {}
-  # default = {
-  #   "ca_cert"    = "none"
-  #   "ca_key"     = "none"
-  #   "root_cert"  = "none"
-  #   "cert_chain" = "none"
-  # }
-  validation {
-    condition     = contains([4, 0], length(compact([for k in ["ca_cert", "ca_key", "root_cert", "cert_chain"] : lookup(var.ca_certs, k, "")])))
-    error_message = "One or more required keys for ca_certs are missing. If you plan to use the self-signed certificate, do not declare the ca_certs variable."
-  }
+variable "enable_mdp" {
+  description = "Determines whether to enable Managed Data Plane (MDP) for this ASM installation."
+  type = bool
+  default = true
 }
 
-variable "iam_member" {
-  description = "The GCP member email address to grant IAM roles to. If impersonate_service_account or service_account is set, roles are granted to that SA."
-  type        = string
-  default     = ""
+variable "enable_cross_cluster_service_discovery" {
+  description = "Determines whether to enable cross-cluster service discovery between this cluster and other clusters in the fleet."
+  type = bool
+  default = false
 }
 
-variable "revision_name" {
-  description = "Sets `--revision-name` option."
-  type        = string
-  default     = "none"
+variable "mesh_config" {
+  description = "MeshConfig specifies configuration available to the control plane. The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig"
+  type = map
+  default = {}
 }

From 912ca06327b98ef183c9c147444aabfe9d6a98ff Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 25 Jan 2022 16:47:40 -0800
Subject: [PATCH 02/42] Move from kubernetes_manifest to bash CPR creation

---
 modules/asm/main.tf                | 52 +++++++++++-------------------
 modules/asm/scripts/create_cpr.sh  | 41 +++++++++++++++++++++++
 modules/asm/scripts/destroy_cpr.sh | 28 ++++++++++++++++
 3 files changed, 88 insertions(+), 33 deletions(-)
 create mode 100755 modules/asm/scripts/create_cpr.sh
 create mode 100755 modules/asm/scripts/destroy_cpr.sh

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index 0049fe0a9e..1f9988ef2c 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -15,7 +15,7 @@
  */
 
 locals {
-  // GKE release channel is a list with max length 1 for some reason https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
+  // GKE release channel is a list with max length 1 https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
   gke_release_channel = length(data.google_container_cluster.asm_cluster.release_channel) > 0 ? data.google_container_cluster.asm_cluster.release_channel[0].channel : ""
   gke_release_channel_fixed = local.gke_release_channel == "UNSPECIFIED" ? "" : local.gke_release_channel
   // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
@@ -30,43 +30,16 @@ data "google_container_cluster" "asm_cluster" {
   location = var.cluster_location
 }
 
-resource "kubernetes_manifest" "cpr" {
-  manifest = {
-    "apiVersion" = "mesh.cloud.google.com/v1beta1"
-    "kind" = "ControlPlaneRevision"
-    "metadata" = {
-      "name" = local.revision_name
-      "namespace" = "istio-system"
-      "labels" = {
-        "mesh.cloud.google.com/managed-cni-enabled" = var.enable_cni
-      }
-    }
-    spec = {
-      type = "managed_service"
-      channel = local.channel
-    }
-  }
-
-  wait_for = {
-    fields = {
-      // In a perfect world we could do wait_for { condition = "Provisioned" } as described in https://github.com/hashicorp/terraform-provider-kubernetes-alpha/issues/12
-      // but it isn't implemented.
-      "status.conditions[0].reason": "Provisioned",
-      "status.conditions[0].status": "True"
-    }
-  }
-
-  timeouts {
-    create = "10m"
+resource "kubernetes_namespace" "system_namespace" {
+  metadata {
+    name = "istio-system"
   }
-
-  depends_on = [kubernetes_config_map.asm_options, kubernetes_config_map.mesh_config]
 }
 
 resource "kubernetes_config_map" "mesh_config" {
     metadata {
       name = local.mesh_config_name
-      namespace = "istio-system"
+      namespace = kubernetes_namespace.system_namespace.metadata[0].name
       annotations = {
         "mesh.cloud.google.com/proxy" = "{\"managed\": \"${var.enable_mdp}\"}"
       }
@@ -82,10 +55,23 @@ resource "kubernetes_config_map" "mesh_config" {
 resource "kubernetes_config_map" "asm_options" {
   metadata {
     name = "asm-options"
-    namespace = "istio-system"
+    namespace = kubernetes_namespace.system_namespace.metadata[0].name
   }
 
   data = {
     CROSS_CLUSTER_SERVICE_DISCOVERY = var.enable_cross_cluster_service_discovery ? "ON" : "OFF"
   }
 }
+
+module "cpr" {
+  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+
+  project_id                  = var.project_id
+  cluster_name                = var.cluster_name
+  cluster_location            = var.cluster_location
+
+  kubectl_create_command  = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${var.enable_cni}"
+  kubectl_destroy_command = "${path.module}/scripts/destroy_cpr.sh ${local.revision_name}"
+
+  module_depends_on = [kubernetes_config_map.asm_options, kubernetes_config_map.mesh_config]
+}
diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
new file mode 100755
index 0000000000..0cf17fb0f7
--- /dev/null
+++ b/modules/asm/scripts/create_cpr.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+if [ "$#" -lt 3 ]; then
+    >&2 echo "Not all expected arguments set."
+    exit 1
+fi
+
+REVISION_NAME=$1; shift
+CHANNEL=$1; shift
+ENABLE_CNI=$1; shift
+
+cat <<EOF | kubectl apply -f -
+apiVersion: mesh.cloud.google.com/v1beta1
+kind: ControlPlaneRevision
+metadata:
+  name: "${REVISION_NAME}"
+  namespace: istio-system
+  labels:
+    mesh.cloud.google.com/managed-cni-enabled: "${ENABLE_CNI}"
+spec:
+  type: managed_service
+  channel: "${CHANNEL}"
+EOF
+
+kubectl wait --for=condition=Reconciled controlplanerevision/asm-managed --timeout 30s
diff --git a/modules/asm/scripts/destroy_cpr.sh b/modules/asm/scripts/destroy_cpr.sh
new file mode 100755
index 0000000000..752c8d8bf2
--- /dev/null
+++ b/modules/asm/scripts/destroy_cpr.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+if [ "$#" -lt 1 ]; then
+    >&2 echo "Not all expected arguments set."
+    exit 1
+fi
+
+REVISION_NAME=$1; shift
+
+if !kubectl delete controlplanerevision -n istio-system ${REVISION_NAME} ; then
+  echo "ControlPlaneRevision ${REVISION_NAME} not found"
+fi

From afed2791507c190585374bedd1b437db2dcf137e Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 25 Jan 2022 18:10:30 -0800
Subject: [PATCH 03/42] Add meshconfig.googleapis.com service

---
 modules/asm/main.tf | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index 1f9988ef2c..e599c9e9c1 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -28,6 +28,22 @@ data "google_container_cluster" "asm_cluster" {
   project = var.project_id
   name = var.cluster_name
   location = var.cluster_location
+
+  // This evaluates during planning phase unless we explicitly require a dependency on
+  // a resource here. This keeps from breaking in cases where we create the GKE cluster and enable
+  // ASM in the same terraform step.
+  depends_on = [kubernetes_namespace.system_namespace]
+}
+
+module "project-services" {
+  source  = "terraform-google-modules/project-factory/google//modules/project_services"
+  version = "~> 10.0"
+
+  project_id    = var.project_id
+  activate_apis = ["meshconfig.googleapis.com"]
+
+  disable_services_on_destroy = false
+  disable_dependent_services  = false
 }
 
 resource "kubernetes_namespace" "system_namespace" {

From 88f76005d252b8e58cd3e64ab875f214c0031973 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 25 Jan 2022 18:37:33 -0800
Subject: [PATCH 04/42] Add namespace to CPR creation script wait

---
 modules/asm/scripts/create_cpr.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index 0cf17fb0f7..00c5897ca4 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -38,4 +38,4 @@ spec:
   channel: "${CHANNEL}"
 EOF
 
-kubectl wait --for=condition=Reconciled controlplanerevision/asm-managed --timeout 30s
+kubectl wait -n istio-system --for=condition=Reconciled controlplanerevision/${REVISION_NAME} --timeout 30s

From 090535988ca27f98bfcc5cb72b3e108d0d718ffb Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 25 Jan 2022 18:56:58 -0800
Subject: [PATCH 05/42] Change example to use google_container_cluster

---
 examples/simple_zonal_with_asm/hub.tf         | 23 +++++--
 examples/simple_zonal_with_asm/main.tf        | 62 +++++++----------
 examples/simple_zonal_with_asm/outputs.tf     | 35 ----------
 .../simple_zonal_with_asm/test_outputs.tf     | 67 -------------------
 modules/asm/scripts/create_cpr.sh             |  3 +
 5 files changed, 45 insertions(+), 145 deletions(-)

diff --git a/examples/simple_zonal_with_asm/hub.tf b/examples/simple_zonal_with_asm/hub.tf
index 4f6162a01c..c184cb3c14 100644
--- a/examples/simple_zonal_with_asm/hub.tf
+++ b/examples/simple_zonal_with_asm/hub.tf
@@ -14,11 +14,20 @@
  * limitations under the License.
  */
 
-module "hub" {
-  source                  = "../../modules/hub"
-  project_id              = var.project_id
-  location                = module.gke.location
-  cluster_name            = module.gke.name
-  cluster_endpoint        = module.gke.endpoint
-  gke_hub_membership_name = "gke-asm-membership"
+resource "google_gke_hub_membership" "cluster_membership" {
+  provider = google-beta
+  project = var.project_id
+  membership_id = "${google_container_cluster.primary.name}-membership"
+  endpoint{
+    gke_cluster {
+      resource_link = "//container.googleapis.com/${google_container_cluster.primary.id}"
+    }
+  }
+}
+# enable Anthos Configmanagement feature on the project.
+resource "google_gke_hub_feature" "mesh_feature" {
+  name = "servicemesh"
+  project = var.project_id
+  location = "global"
+  provider = google-beta
 }
diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf
index 849248d38e..43c04c2f6b 100644
--- a/examples/simple_zonal_with_asm/main.tf
+++ b/examples/simple_zonal_with_asm/main.tf
@@ -21,52 +21,42 @@ locals {
 data "google_client_config" "default" {}
 
 provider "kubernetes" {
-  host                   = "https://${module.gke.endpoint}"
+  host                   = "https://${google_container_cluster.primary.endpoint}"
   token                  = data.google_client_config.default.access_token
-  cluster_ca_certificate = base64decode(module.gke.ca_certificate)
+  cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth[0].cluster_ca_certificate)
 }
 
 data "google_project" "project" {
   project_id = var.project_id
 }
 
-module "gke" {
-  source                  = "../../"
-  project_id              = var.project_id
-  name                    = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
-  regional                = false
-  region                  = var.region
-  zones                   = var.zones
-  release_channel         = "REGULAR"
-  network                 = var.network
-  subnetwork              = var.subnetwork
-  ip_range_pods           = var.ip_range_pods
-  ip_range_services       = var.ip_range_services
-  network_policy          = false
-  cluster_resource_labels = { "mesh_id" : "proj-${data.google_project.project.number}" }
-  node_pools = [
-    {
-      name         = "asm-node-pool"
-      autoscaling  = false
-      auto_upgrade = true
-      # ASM requires minimum 4 nodes and e2-standard-4
-      node_count   = 4
-      machine_type = "e2-standard-4"
-    },
-  ]
+resource "google_container_cluster" "primary" {
+  name               = "drew-barrymore"
+  project = var.project_id
+  location           = "us-central1-a"
+  initial_node_count = 3
+  workload_identity_config {
+    identity_namespace = "${var.project_id}.svc.id.goog"
+  }
+  node_config {
+    # Google recommends custom service accounts that have cloud-platform scope and permissions granted via IAM Roles.
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/cloud-platform"
+    ]
+    labels = {
+      foo = "bar"
+    }
+    tags = ["foo", "bar"]
+  }
+  timeouts {
+    create = "30m"
+    update = "40m"
+  }
 }
 
 module "asm" {
   source                    = "../../modules/asm"
-  cluster_name              = module.gke.name
-  cluster_endpoint          = module.gke.endpoint
   project_id                = var.project_id
-  location                  = module.gke.location
-  enable_cluster_roles      = true
-  enable_cluster_labels     = true
-  enable_gcp_apis           = true
-  enable_gcp_components     = true
-  enable_namespace_creation = true
-  options                   = ["envoy-access-log"]
-  outdir                    = "./${module.gke.name}-outdir"
+  cluster_name              = google_container_cluster.primary.name
+  cluster_location          = google_container_cluster.primary.location
 }
diff --git a/examples/simple_zonal_with_asm/outputs.tf b/examples/simple_zonal_with_asm/outputs.tf
index 1e9b7913da..e69de29bb2 100644
--- a/examples/simple_zonal_with_asm/outputs.tf
+++ b/examples/simple_zonal_with_asm/outputs.tf
@@ -1,35 +0,0 @@
-/**
- * Copyright 2018 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-output "kubernetes_endpoint" {
-  sensitive = true
-  value     = module.gke.endpoint
-}
-
-output "client_token" {
-  sensitive = true
-  value     = base64encode(data.google_client_config.default.access_token)
-}
-
-output "ca_certificate" {
-  sensitive = true
-  value     = module.gke.ca_certificate
-}
-
-output "service_account" {
-  description = "The default service account used for running nodes."
-  value       = module.gke.service_account
-}
diff --git a/examples/simple_zonal_with_asm/test_outputs.tf b/examples/simple_zonal_with_asm/test_outputs.tf
index 71e5965e05..e69de29bb2 100644
--- a/examples/simple_zonal_with_asm/test_outputs.tf
+++ b/examples/simple_zonal_with_asm/test_outputs.tf
@@ -1,67 +0,0 @@
-/**
- * Copyright 2018 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// These outputs are used to test the module with kitchen-terraform
-// They do not need to be included in real-world uses of this module
-
-output "project_id" {
-  value = var.project_id
-}
-
-output "region" {
-  value = module.gke.region
-}
-
-output "cluster_name" {
-  description = "Cluster name"
-  value       = module.gke.name
-}
-
-output "network" {
-  value = var.network
-}
-
-output "subnetwork" {
-  value = var.subnetwork
-}
-
-output "location" {
-  value = module.gke.location
-}
-
-output "ip_range_pods" {
-  description = "The secondary IP range used for pods"
-  value       = var.ip_range_pods
-}
-
-output "ip_range_services" {
-  description = "The secondary IP range used for services"
-  value       = var.ip_range_services
-}
-
-output "zones" {
-  description = "List of zones in which the cluster resides"
-  value       = module.gke.zones
-}
-
-output "master_kubernetes_version" {
-  description = "The master Kubernetes version"
-  value       = module.gke.master_version
-}
-
-output "identity_namespace" {
-  value = module.gke.identity_namespace
-}
diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index 00c5897ca4..22e7d6753d 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -21,6 +21,9 @@ if [ "$#" -lt 3 ]; then
     exit 1
 fi
 
+echo "Sleeping for CPR... do retries instead..."
+sleep 30
+
 REVISION_NAME=$1; shift
 CHANNEL=$1; shift
 ENABLE_CNI=$1; shift

From 8538e869dd1fb58a524dbe87360a0f2e84c62ed2 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 25 Jan 2022 19:41:16 -0800
Subject: [PATCH 06/42] Improve CPR status wait duration

---
 modules/asm/scripts/create_cpr.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index 22e7d6753d..912d6128e8 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -41,4 +41,4 @@ spec:
   channel: "${CHANNEL}"
 EOF
 
-kubectl wait -n istio-system --for=condition=Reconciled controlplanerevision/${REVISION_NAME} --timeout 30s
+kubectl wait -n istio-system --for=condition=Reconciled controlplanerevision/${REVISION_NAME} --timeout 5m

From 11f9d905f0bf410db49d84f8bd1d9c05ea22708a Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 1 Feb 2022 17:27:04 -0800
Subject: [PATCH 07/42] Use retries rather than sleeping to wait for CPR CRD
 existence

---
 modules/asm/scripts/create_cpr.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index 912d6128e8..dc706757a9 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -21,8 +21,10 @@ if [ "$#" -lt 3 ]; then
     exit 1
 fi
 
-echo "Sleeping for CPR... do retries instead..."
-sleep 30
+# Wait for the CRD to get created before creating the CPR.
+readonly CPR_RESOURCE=controlplanerevisions.mesh.cloud.google.com
+for i in {1..6}; do kubectl get crd ${CPR_RESOURCE} && break || sleep 10; done
+kubectl wait --for condition=established --timeout=60s crd/${CPR_RESOURCE}
 
 REVISION_NAME=$1; shift
 CHANNEL=$1; shift

From 56240f109192197d54895f3c83447a589a1c005c Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 1 Feb 2022 17:27:57 -0800
Subject: [PATCH 08/42] Enable servicemesh feature in module

---
 modules/asm/main.tf | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index e599c9e9c1..f4a72a0b7c 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -35,17 +35,6 @@ data "google_container_cluster" "asm_cluster" {
   depends_on = [kubernetes_namespace.system_namespace]
 }
 
-module "project-services" {
-  source  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version = "~> 10.0"
-
-  project_id    = var.project_id
-  activate_apis = ["meshconfig.googleapis.com"]
-
-  disable_services_on_destroy = false
-  disable_dependent_services  = false
-}
-
 resource "kubernetes_namespace" "system_namespace" {
   metadata {
     name = "istio-system"
@@ -79,6 +68,13 @@ resource "kubernetes_config_map" "asm_options" {
   }
 }
 
+resource "google_gke_hub_feature" "mesh_feature" {
+  name = "servicemesh"
+  project = var.project_id
+  location = "global"
+  provider = google-beta
+}
+
 module "cpr" {
   source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
 
@@ -91,3 +87,14 @@ module "cpr" {
 
   module_depends_on = [kubernetes_config_map.asm_options, kubernetes_config_map.mesh_config]
 }
+
+module "project-services" {
+  source  = "terraform-google-modules/project-factory/google//modules/project_services"
+  version = "~> 10.0"
+
+  project_id    = var.project_id
+  activate_apis = ["mesh.googleapis.com"]
+
+  disable_services_on_destroy = false
+  disable_dependent_services  = false
+}

From 5235884d2b2cf621a7f8812b9d470797e377db85 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 1 Feb 2022 17:31:43 -0800
Subject: [PATCH 09/42] Revert changes to examples

---
 examples/simple_zonal_with_asm/hub.tf         | 11 +--
 examples/simple_zonal_with_asm/main.tf        | 55 ++++++++-------
 examples/simple_zonal_with_asm/outputs.tf     | 35 ++++++++++
 .../simple_zonal_with_asm/test_outputs.tf     | 67 +++++++++++++++++++
 4 files changed, 133 insertions(+), 35 deletions(-)

diff --git a/examples/simple_zonal_with_asm/hub.tf b/examples/simple_zonal_with_asm/hub.tf
index c184cb3c14..2cbfd291e8 100644
--- a/examples/simple_zonal_with_asm/hub.tf
+++ b/examples/simple_zonal_with_asm/hub.tf
@@ -17,17 +17,10 @@
 resource "google_gke_hub_membership" "cluster_membership" {
   provider = google-beta
   project = var.project_id
-  membership_id = "${google_container_cluster.primary.name}-membership"
+  membership_id = "${module.gke.name}-membership"
   endpoint{
     gke_cluster {
-      resource_link = "//container.googleapis.com/${google_container_cluster.primary.id}"
+      resource_link = "//container.googleapis.com/${module.gke.cluster_id}"
     }
   }
 }
-# enable Anthos Configmanagement feature on the project.
-resource "google_gke_hub_feature" "mesh_feature" {
-  name = "servicemesh"
-  project = var.project_id
-  location = "global"
-  provider = google-beta
-}
diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf
index 43c04c2f6b..4538268116 100644
--- a/examples/simple_zonal_with_asm/main.tf
+++ b/examples/simple_zonal_with_asm/main.tf
@@ -21,42 +21,45 @@ locals {
 data "google_client_config" "default" {}
 
 provider "kubernetes" {
-  host                   = "https://${google_container_cluster.primary.endpoint}"
+  host                   = "https://${module.gke.endpoint}"
   token                  = data.google_client_config.default.access_token
-  cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth[0].cluster_ca_certificate)
+  cluster_ca_certificate = base64decode(module.gke.ca_certificate)
 }
 
 data "google_project" "project" {
   project_id = var.project_id
 }
 
-resource "google_container_cluster" "primary" {
-  name               = "drew-barrymore"
-  project = var.project_id
-  location           = "us-central1-a"
-  initial_node_count = 3
-  workload_identity_config {
-    identity_namespace = "${var.project_id}.svc.id.goog"
-  }
-  node_config {
-    # Google recommends custom service accounts that have cloud-platform scope and permissions granted via IAM Roles.
-    oauth_scopes = [
-      "https://www.googleapis.com/auth/cloud-platform"
-    ]
-    labels = {
-      foo = "bar"
-    }
-    tags = ["foo", "bar"]
-  }
-  timeouts {
-    create = "30m"
-    update = "40m"
-  }
+module "gke" {
+  source                  = "../../"
+  project_id              = var.project_id
+  name                    = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
+  regional                = false
+  region                  = var.region
+  zones                   = var.zones
+  release_channel         = "REGULAR"
+  network                 = var.network
+  subnetwork              = var.subnetwork
+  ip_range_pods           = var.ip_range_pods
+  ip_range_services       = var.ip_range_services
+  network_policy          = false
+  cluster_resource_labels = { "mesh_id" : "proj-${data.google_project.project.number}" }
+  node_pools = [
+    {
+      name         = "asm-node-pool"
+      autoscaling  = false
+      auto_upgrade = true
+      # ASM requires minimum 4 nodes and e2-standard-4
+      node_count   = 4
+      machine_type = "e2-standard-4"
+    },
+  ]
 }
 
 module "asm" {
   source                    = "../../modules/asm"
+  cluster_name              = module.gke.name
+  cluster_location          = module.gke.location
   project_id                = var.project_id
-  cluster_name              = google_container_cluster.primary.name
-  cluster_location          = google_container_cluster.primary.location
+  location                  = module.gke.location
 }
diff --git a/examples/simple_zonal_with_asm/outputs.tf b/examples/simple_zonal_with_asm/outputs.tf
index e69de29bb2..1e9b7913da 100644
--- a/examples/simple_zonal_with_asm/outputs.tf
+++ b/examples/simple_zonal_with_asm/outputs.tf
@@ -0,0 +1,35 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "kubernetes_endpoint" {
+  sensitive = true
+  value     = module.gke.endpoint
+}
+
+output "client_token" {
+  sensitive = true
+  value     = base64encode(data.google_client_config.default.access_token)
+}
+
+output "ca_certificate" {
+  sensitive = true
+  value     = module.gke.ca_certificate
+}
+
+output "service_account" {
+  description = "The default service account used for running nodes."
+  value       = module.gke.service_account
+}
diff --git a/examples/simple_zonal_with_asm/test_outputs.tf b/examples/simple_zonal_with_asm/test_outputs.tf
index e69de29bb2..71e5965e05 100644
--- a/examples/simple_zonal_with_asm/test_outputs.tf
+++ b/examples/simple_zonal_with_asm/test_outputs.tf
@@ -0,0 +1,67 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// These outputs are used to test the module with kitchen-terraform
+// They do not need to be included in real-world uses of this module
+
+output "project_id" {
+  value = var.project_id
+}
+
+output "region" {
+  value = module.gke.region
+}
+
+output "cluster_name" {
+  description = "Cluster name"
+  value       = module.gke.name
+}
+
+output "network" {
+  value = var.network
+}
+
+output "subnetwork" {
+  value = var.subnetwork
+}
+
+output "location" {
+  value = module.gke.location
+}
+
+output "ip_range_pods" {
+  description = "The secondary IP range used for pods"
+  value       = var.ip_range_pods
+}
+
+output "ip_range_services" {
+  description = "The secondary IP range used for services"
+  value       = var.ip_range_services
+}
+
+output "zones" {
+  description = "List of zones in which the cluster resides"
+  value       = module.gke.zones
+}
+
+output "master_kubernetes_version" {
+  description = "The master Kubernetes version"
+  value       = module.gke.master_version
+}
+
+output "identity_namespace" {
+  value = module.gke.identity_namespace
+}

From c61787a6b5bcafc7d991295fbe144eb40395a2d4 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 1 Feb 2022 17:42:16 -0800
Subject: [PATCH 10/42] Fix enable_mdp to just enable CNI

---
 modules/asm/main.tf      | 9 ++++-----
 modules/asm/variables.tf | 2 ++
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index f4a72a0b7c..cef31f2f3a 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -21,7 +21,9 @@ locals {
   // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
   channel = lower(coalesce(var.channel, local.gke_release_channel_fixed, "regular"))
   revision_name = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
-  mesh_config_name= "istio-${local.revision_name}"
+  mesh_config_name = "istio-${local.revision_name}"
+  // CNI should be enabled if either enable_cni or enable_mdp are set
+  enable_cni = var.enable_cni || var.enable_mdp
 }
 
 data "google_container_cluster" "asm_cluster" {
@@ -45,9 +47,6 @@ resource "kubernetes_config_map" "mesh_config" {
     metadata {
       name = local.mesh_config_name
       namespace = kubernetes_namespace.system_namespace.metadata[0].name
-      annotations = {
-        "mesh.cloud.google.com/proxy" = "{\"managed\": \"${var.enable_mdp}\"}"
-      }
       labels = {
         "istio.io/rev" = local.revision_name
       }
@@ -82,7 +81,7 @@ module "cpr" {
   cluster_name                = var.cluster_name
   cluster_location            = var.cluster_location
 
-  kubectl_create_command  = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${var.enable_cni}"
+  kubectl_create_command  = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${local.enable_cni}"
   kubectl_destroy_command = "${path.module}/scripts/destroy_cpr.sh ${local.revision_name}"
 
   module_depends_on = [kubernetes_config_map.asm_options, kubernetes_config_map.mesh_config]
diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf
index 6aab28a5ee..54223325ec 100644
--- a/modules/asm/variables.tf
+++ b/modules/asm/variables.tf
@@ -50,6 +50,8 @@ variable "enable_cni" {
   default = true
 }
 
+// This should be validated so that it cannot be enabled while CNI is disabled
+// but validating based on other variables is not possible today (https://github.com/hashicorp/terraform/issues/25609)
 variable "enable_mdp" {
   description = "Determines whether to enable Managed Data Plane (MDP) for this ASM installation."
   type = bool

From f5b65bd5f4c4da81805363f82432b6e4f7a1a945 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 1 Feb 2022 18:19:58 -0800
Subject: [PATCH 11/42] Lint

---
 examples/simple_zonal_with_asm/hub.tf  |  6 ++--
 examples/simple_zonal_with_asm/main.tf | 10 +++---
 modules/asm/README.md                  |  2 +-
 modules/asm/main.tf                    | 42 +++++++++++++-------------
 modules/asm/outputs.tf                 |  2 +-
 modules/asm/scripts/create_cpr.sh      | 15 +++++++--
 modules/asm/scripts/destroy_cpr.sh     |  2 +-
 modules/asm/variables.tf               | 24 +++++++--------
 8 files changed, 56 insertions(+), 47 deletions(-)

diff --git a/examples/simple_zonal_with_asm/hub.tf b/examples/simple_zonal_with_asm/hub.tf
index 2cbfd291e8..79d9fdb2b6 100644
--- a/examples/simple_zonal_with_asm/hub.tf
+++ b/examples/simple_zonal_with_asm/hub.tf
@@ -15,10 +15,10 @@
  */
 
 resource "google_gke_hub_membership" "cluster_membership" {
-  provider = google-beta
-  project = var.project_id
+  provider      = google-beta
+  project       = var.project_id
   membership_id = "${module.gke.name}-membership"
-  endpoint{
+  endpoint {
     gke_cluster {
       resource_link = "//container.googleapis.com/${module.gke.cluster_id}"
     }
diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf
index 4538268116..a1f30659f5 100644
--- a/examples/simple_zonal_with_asm/main.tf
+++ b/examples/simple_zonal_with_asm/main.tf
@@ -57,9 +57,9 @@ module "gke" {
 }
 
 module "asm" {
-  source                    = "../../modules/asm"
-  cluster_name              = module.gke.name
-  cluster_location          = module.gke.location
-  project_id                = var.project_id
-  location                  = module.gke.location
+  source           = "../../modules/asm"
+  cluster_name     = module.gke.name
+  cluster_location = module.gke.location
+  project_id       = var.project_id
+  location         = module.gke.location
 }
diff --git a/modules/asm/README.md b/modules/asm/README.md
index e95e20fd91..096805bc23 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -15,7 +15,7 @@ This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh
 | enable\_cni | Determines whether to enable CNI for this ASM installation. | `bool` | `true` | no |
 | enable\_cross\_cluster\_service\_discovery | Determines whether to enable cross-cluster service discovery between this cluster and other clusters in the fleet. | `bool` | `false` | no |
 | enable\_mdp | Determines whether to enable Managed Data Plane (MDP) for this ASM installation. | `bool` | `true` | no |
-| mesh\_config | MeshConfig specifies configuration available to the control plane. The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig | `map` | `{}` | no |
+| mesh\_config | MeshConfig specifies configuration available to the control plane. The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig | `map(any)` | `{}` | no |
 | project\_id | The project in which the resource belongs. | `string` | n/a | yes |
 
 ## Outputs
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index cef31f2f3a..473a34b70b 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -16,19 +16,19 @@
 
 locals {
   // GKE release channel is a list with max length 1 https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
-  gke_release_channel = length(data.google_container_cluster.asm_cluster.release_channel) > 0 ? data.google_container_cluster.asm_cluster.release_channel[0].channel : ""
+  gke_release_channel       = length(data.google_container_cluster.asm_cluster.release_channel) > 0 ? data.google_container_cluster.asm_cluster.release_channel[0].channel : ""
   gke_release_channel_fixed = local.gke_release_channel == "UNSPECIFIED" ? "" : local.gke_release_channel
   // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
-  channel = lower(coalesce(var.channel, local.gke_release_channel_fixed, "regular"))
-  revision_name = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
+  channel          = lower(coalesce(var.channel, local.gke_release_channel_fixed, "regular"))
+  revision_name    = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
   mesh_config_name = "istio-${local.revision_name}"
   // CNI should be enabled if either enable_cni or enable_mdp are set
   enable_cni = var.enable_cni || var.enable_mdp
 }
 
 data "google_container_cluster" "asm_cluster" {
-  project = var.project_id
-  name = var.cluster_name
+  project  = var.project_id
+  name     = var.cluster_name
   location = var.cluster_location
 
   // This evaluates during planning phase unless we explicitly require a dependency on
@@ -44,21 +44,21 @@ resource "kubernetes_namespace" "system_namespace" {
 }
 
 resource "kubernetes_config_map" "mesh_config" {
-    metadata {
-      name = local.mesh_config_name
-      namespace = kubernetes_namespace.system_namespace.metadata[0].name
-      labels = {
-        "istio.io/rev" = local.revision_name
-      }
-    }
-    data = {
-      mesh = yamlencode(var.mesh_config)
+  metadata {
+    name      = local.mesh_config_name
+    namespace = kubernetes_namespace.system_namespace.metadata[0].name
+    labels = {
+      "istio.io/rev" = local.revision_name
     }
+  }
+  data = {
+    mesh = yamlencode(var.mesh_config)
+  }
 }
 
 resource "kubernetes_config_map" "asm_options" {
   metadata {
-    name = "asm-options"
+    name      = "asm-options"
     namespace = kubernetes_namespace.system_namespace.metadata[0].name
   }
 
@@ -68,18 +68,18 @@ resource "kubernetes_config_map" "asm_options" {
 }
 
 resource "google_gke_hub_feature" "mesh_feature" {
-  name = "servicemesh"
-  project = var.project_id
+  name     = "servicemesh"
+  project  = var.project_id
   location = "global"
   provider = google-beta
 }
 
 module "cpr" {
-  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+  source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
 
-  project_id                  = var.project_id
-  cluster_name                = var.cluster_name
-  cluster_location            = var.cluster_location
+  project_id       = var.project_id
+  cluster_name     = var.cluster_name
+  cluster_location = var.cluster_location
 
   kubectl_create_command  = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${local.enable_cni}"
   kubectl_destroy_command = "${path.module}/scripts/destroy_cpr.sh ${local.revision_name}"
diff --git a/modules/asm/outputs.tf b/modules/asm/outputs.tf
index 969ff8699c..4a82664e70 100644
--- a/modules/asm/outputs.tf
+++ b/modules/asm/outputs.tf
@@ -15,6 +15,6 @@
  */
 
 output "revision_name" {
-  value = local.revision_name
+  value       = local.revision_name
   description = "The name of the installed managed ASM revision."
 }
diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index dc706757a9..2cf074b84c 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -23,8 +23,17 @@ fi
 
 # Wait for the CRD to get created before creating the CPR.
 readonly CPR_RESOURCE=controlplanerevisions.mesh.cloud.google.com
-for i in {1..6}; do kubectl get crd ${CPR_RESOURCE} && break || sleep 10; done
-kubectl wait --for condition=established --timeout=60s crd/${CPR_RESOURCE}
+for _i in {1..6}; do
+  echo "Ensuring ControlPlaneRevision exists in cluster... attempt ${_i}"
+  if kubectl get crd "${CPR_RESOURCE}"
+  then
+    break
+  else
+    sleep 10
+  fi
+done
+
+kubectl wait --for condition=established --timeout=60s crd/"${CPR_RESOURCE}"
 
 REVISION_NAME=$1; shift
 CHANNEL=$1; shift
@@ -43,4 +52,4 @@ spec:
   channel: "${CHANNEL}"
 EOF
 
-kubectl wait -n istio-system --for=condition=Reconciled controlplanerevision/${REVISION_NAME} --timeout 5m
+kubectl wait -n istio-system --for=condition=Reconciled controlplanerevision/"${REVISION_NAME}" --timeout 5m
diff --git a/modules/asm/scripts/destroy_cpr.sh b/modules/asm/scripts/destroy_cpr.sh
index 752c8d8bf2..6c8b4f8e71 100755
--- a/modules/asm/scripts/destroy_cpr.sh
+++ b/modules/asm/scripts/destroy_cpr.sh
@@ -23,6 +23,6 @@ fi
 
 REVISION_NAME=$1; shift
 
-if !kubectl delete controlplanerevision -n istio-system ${REVISION_NAME} ; then
+if ! kubectl delete controlplanerevision -n istio-system "${REVISION_NAME}" ; then
   echo "ControlPlaneRevision ${REVISION_NAME} not found"
 fi
diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf
index 54223325ec..7885ceea29 100644
--- a/modules/asm/variables.tf
+++ b/modules/asm/variables.tf
@@ -16,22 +16,22 @@
 
 variable "project_id" {
   description = "The project in which the resource belongs."
-  type = string
+  type        = string
 }
 
 variable "cluster_name" {
   description = "The unique name to identify the cluster in ASM."
-  type = string
+  type        = string
 }
 
 variable "cluster_location" {
   description = "The cluster location for this ASM installation."
-  type = string
+  type        = string
 }
 
 variable "channel" {
   description = "The channel to use for this ASM installation."
-  type = string
+  type        = string
   validation {
     condition = anytrue([
       var.channel == "rapid",
@@ -46,26 +46,26 @@ variable "channel" {
 
 variable "enable_cni" {
   description = "Determines whether to enable CNI for this ASM installation."
-  type = bool
-  default = true
+  type        = bool
+  default     = true
 }
 
 // This should be validated so that it cannot be enabled while CNI is disabled
 // but validating based on other variables is not possible today (https://github.com/hashicorp/terraform/issues/25609)
 variable "enable_mdp" {
   description = "Determines whether to enable Managed Data Plane (MDP) for this ASM installation."
-  type = bool
-  default = true
+  type        = bool
+  default     = true
 }
 
 variable "enable_cross_cluster_service_discovery" {
   description = "Determines whether to enable cross-cluster service discovery between this cluster and other clusters in the fleet."
-  type = bool
-  default = false
+  type        = bool
+  default     = false
 }
 
 variable "mesh_config" {
   description = "MeshConfig specifies configuration available to the control plane. The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig"
-  type = map
-  default = {}
+  type        = map(any)
+  default     = {}
 }

From 4818dc651d31effc66b42095228769887560f973 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Mon, 7 Feb 2022 11:45:46 -0800
Subject: [PATCH 12/42] minor fixes

---
 modules/asm/main.tf               | 34 +++++++++----------------------
 modules/asm/scripts/create_cpr.sh |  4 ++++
 modules/asm/variables.tf          |  4 ++--
 3 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index 473a34b70b..0eddd186d4 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -16,7 +16,7 @@
 
 locals {
   // GKE release channel is a list with max length 1 https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
-  gke_release_channel       = length(data.google_container_cluster.asm_cluster.release_channel) > 0 ? data.google_container_cluster.asm_cluster.release_channel[0].channel : ""
+  gke_release_channel       = data.google_container_cluster.asm_cluster.release_channel != null ? data.google_container_cluster.asm_cluster.release_channel[0].channel : ""
   gke_release_channel_fixed = local.gke_release_channel == "UNSPECIFIED" ? "" : local.gke_release_channel
   // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
   channel          = lower(coalesce(var.channel, local.gke_release_channel_fixed, "regular"))
@@ -30,11 +30,6 @@ data "google_container_cluster" "asm_cluster" {
   project  = var.project_id
   name     = var.cluster_name
   location = var.cluster_location
-
-  // This evaluates during planning phase unless we explicitly require a dependency on
-  // a resource here. This keeps from breaking in cases where we create the GKE cluster and enable
-  // ASM in the same terraform step.
-  depends_on = [kubernetes_namespace.system_namespace]
 }
 
 resource "kubernetes_namespace" "system_namespace" {
@@ -67,12 +62,14 @@ resource "kubernetes_config_map" "asm_options" {
   }
 }
 
-resource "google_gke_hub_feature" "mesh_feature" {
-  name     = "servicemesh"
-  project  = var.project_id
-  location = "global"
-  provider = google-beta
-}
+# TODO(Monkeyanator) due to a bug in the gke_hub_feature resource implementation this fails when enabling an enabled
+# feature (i.e. running the module twice) and does not disable the feature upon TF destroy.
+#resource "google_gke_hub_feature" "mesh_feature" {
+#  name     = "servicemesh"
+#  project  = var.project_id
+#  location = "global"
+#  provider = google-beta
+#}
 
 module "cpr" {
   source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
@@ -84,16 +81,5 @@ module "cpr" {
   kubectl_create_command  = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${local.enable_cni}"
   kubectl_destroy_command = "${path.module}/scripts/destroy_cpr.sh ${local.revision_name}"
 
-  module_depends_on = [kubernetes_config_map.asm_options, kubernetes_config_map.mesh_config]
-}
-
-module "project-services" {
-  source  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version = "~> 10.0"
-
-  project_id    = var.project_id
-  activate_apis = ["mesh.googleapis.com"]
-
-  disable_services_on_destroy = false
-  disable_dependent_services  = false
+  module_depends_on = [kubernetes_config_map.mesh_config, kubernetes_config_map.asm_options]
 }
diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index 2cf074b84c..5e22f615e7 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -35,6 +35,10 @@ done
 
 kubectl wait --for condition=established --timeout=60s crd/"${CPR_RESOURCE}"
 
+if ! kubectl create namespace istio-system; then
+  echo "Failed to create system namespace; continuing since this can indicate existence"
+fi
+
 REVISION_NAME=$1; shift
 CHANNEL=$1; shift
 ENABLE_CNI=$1; shift
diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf
index 7885ceea29..d0481c4fe4 100644
--- a/modules/asm/variables.tf
+++ b/modules/asm/variables.tf
@@ -47,7 +47,7 @@ variable "channel" {
 variable "enable_cni" {
   description = "Determines whether to enable CNI for this ASM installation."
   type        = bool
-  default     = true
+  default     = false
 }
 
 // This should be validated so that it cannot be enabled while CNI is disabled
@@ -55,7 +55,7 @@ variable "enable_cni" {
 variable "enable_mdp" {
   description = "Determines whether to enable Managed Data Plane (MDP) for this ASM installation."
   type        = bool
-  default     = true
+  default     = false
 }
 
 variable "enable_cross_cluster_service_discovery" {

From 8ce063555b74b4e64878092e3a0fae824f979ef4 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 8 Feb 2022 19:05:06 -0800
Subject: [PATCH 13/42] Bump timeout on status wait

---
 modules/asm/scripts/create_cpr.sh | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index 5e22f615e7..5bd7163f34 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -23,7 +23,7 @@ fi
 
 # Wait for the CRD to get created before creating the CPR.
 readonly CPR_RESOURCE=controlplanerevisions.mesh.cloud.google.com
-for _i in {1..6}; do
+for _i in {1..18}; do
   echo "Ensuring ControlPlaneRevision exists in cluster... attempt ${_i}"
   if kubectl get crd "${CPR_RESOURCE}"
   then
@@ -35,10 +35,6 @@ done
 
 kubectl wait --for condition=established --timeout=60s crd/"${CPR_RESOURCE}"
 
-if ! kubectl create namespace istio-system; then
-  echo "Failed to create system namespace; continuing since this can indicate existence"
-fi
-
 REVISION_NAME=$1; shift
 CHANNEL=$1; shift
 ENABLE_CNI=$1; shift
@@ -56,4 +52,4 @@ spec:
   channel: "${CHANNEL}"
 EOF
 
-kubectl wait -n istio-system --for=condition=Reconciled controlplanerevision/"${REVISION_NAME}" --timeout 5m
+kubectl wait -n istio-system --for=condition=Reconciled controlplanerevision/"${REVISION_NAME}" --timeout 10m

From c961aaa20adf1c4ad0289d990012b6ed88055fac Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Thu, 10 Feb 2022 10:16:22 -0800
Subject: [PATCH 14/42] Don't create MeshConfig if unset

---
 modules/asm/main.tf      | 6 +++++-
 modules/asm/variables.tf | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index 0eddd186d4..e1873bd03d 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -39,6 +39,8 @@ resource "kubernetes_namespace" "system_namespace" {
 }
 
 resource "kubernetes_config_map" "mesh_config" {
+  count = length(var.mesh_config) == 0 ? 0 : 1
+
   metadata {
     name      = local.mesh_config_name
     namespace = kubernetes_namespace.system_namespace.metadata[0].name
@@ -46,6 +48,7 @@ resource "kubernetes_config_map" "mesh_config" {
       "istio.io/rev" = local.revision_name
     }
   }
+
   data = {
     mesh = yamlencode(var.mesh_config)
   }
@@ -55,7 +58,8 @@ resource "kubernetes_config_map" "asm_options" {
   metadata {
     name      = "asm-options"
     namespace = kubernetes_namespace.system_namespace.metadata[0].name
-  }
+}
+
 
   data = {
     CROSS_CLUSTER_SERVICE_DISCOVERY = var.enable_cross_cluster_service_discovery ? "ON" : "OFF"
diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf
index d0481c4fe4..76a8721fdd 100644
--- a/modules/asm/variables.tf
+++ b/modules/asm/variables.tf
@@ -65,7 +65,7 @@ variable "enable_cross_cluster_service_discovery" {
 }
 
 variable "mesh_config" {
-  description = "MeshConfig specifies configuration available to the control plane. The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig"
+  description = "MeshConfig specifies configuration available to the control plane. If unset the module will not attempt to create the MeshConfig (i.e. if managing this configuration elsewhere). The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig"
   type        = map(any)
   default     = {}
 }

From 7ad8320b7fefe5d99b45b281ff7581a1f2f63209 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Thu, 10 Feb 2022 15:40:01 -0800
Subject: [PATCH 15/42] Fix ASM sample

---
 examples/simple_zonal_with_asm/hub.tf  | 8 ++++++++
 examples/simple_zonal_with_asm/main.tf | 5 +++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/examples/simple_zonal_with_asm/hub.tf b/examples/simple_zonal_with_asm/hub.tf
index 79d9fdb2b6..036f360adf 100644
--- a/examples/simple_zonal_with_asm/hub.tf
+++ b/examples/simple_zonal_with_asm/hub.tf
@@ -14,6 +14,13 @@
  * limitations under the License.
  */
 
+resource "google_gke_hub_feature" "mesh_feature" {
+  name     = "servicemesh"
+  project  = var.project_id
+  location = "global"
+  provider = google-beta
+}
+
 resource "google_gke_hub_membership" "cluster_membership" {
   provider      = google-beta
   project       = var.project_id
@@ -24,3 +31,4 @@ resource "google_gke_hub_membership" "cluster_membership" {
     }
   }
 }
+
diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf
index a1f30659f5..61bbeae946 100644
--- a/examples/simple_zonal_with_asm/main.tf
+++ b/examples/simple_zonal_with_asm/main.tf
@@ -58,8 +58,9 @@ module "gke" {
 
 module "asm" {
   source           = "../../modules/asm"
+  project_id       = var.project_id
   cluster_name     = module.gke.name
   cluster_location = module.gke.location
-  project_id       = var.project_id
-  location         = module.gke.location
+  enable_cross_cluster_service_discovery = true
+  enable_cni = true
 }

From e4e81d666ef8160ea568bf786c75a55b69af6731 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Thu, 10 Feb 2022 16:18:25 -0800
Subject: [PATCH 16/42] Update README autogen

---
 modules/asm/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/asm/README.md b/modules/asm/README.md
index 096805bc23..2e7564915c 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -12,10 +12,10 @@ This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh
 | channel | The channel to use for this ASM installation. | `string` | `""` | no |
 | cluster\_location | The cluster location for this ASM installation. | `string` | n/a | yes |
 | cluster\_name | The unique name to identify the cluster in ASM. | `string` | n/a | yes |
-| enable\_cni | Determines whether to enable CNI for this ASM installation. | `bool` | `true` | no |
+| enable\_cni | Determines whether to enable CNI for this ASM installation. | `bool` | `false` | no |
 | enable\_cross\_cluster\_service\_discovery | Determines whether to enable cross-cluster service discovery between this cluster and other clusters in the fleet. | `bool` | `false` | no |
-| enable\_mdp | Determines whether to enable Managed Data Plane (MDP) for this ASM installation. | `bool` | `true` | no |
-| mesh\_config | MeshConfig specifies configuration available to the control plane. The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig | `map(any)` | `{}` | no |
+| enable\_mdp | Determines whether to enable Managed Data Plane (MDP) for this ASM installation. | `bool` | `false` | no |
+| mesh\_config | MeshConfig specifies configuration available to the control plane. If unset the module will not attempt to create the MeshConfig (i.e. if managing this configuration elsewhere). The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig | `map(any)` | `{}` | no |
 | project\_id | The project in which the resource belongs. | `string` | n/a | yes |
 
 ## Outputs

From 7f1b0e269493996b838baca4f0079d3d80d1eea9 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Fri, 11 Feb 2022 17:04:39 -0800
Subject: [PATCH 17/42] Minor fixes

---
 modules/asm/main.tf               | 16 +++++++---------
 modules/asm/scripts/create_cpr.sh | 11 ++++++-----
 2 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index e1873bd03d..cfc38f2b4c 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -16,23 +16,21 @@
 
 locals {
   // GKE release channel is a list with max length 1 https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
-  gke_release_channel       = data.google_container_cluster.asm_cluster.release_channel != null ? data.google_container_cluster.asm_cluster.release_channel[0].channel : ""
-  gke_release_channel_fixed = local.gke_release_channel == "UNSPECIFIED" ? "" : local.gke_release_channel
+  gke_release_channel       = data.google_container_cluster.asm.release_channel != null ? data.google_container_cluster.asm.release_channel[0].channel : ""
   // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
-  channel          = lower(coalesce(var.channel, local.gke_release_channel_fixed, "regular"))
+  channel          = lower(coalesce(var.channel, local.gke_release_channel, "regular"))
   revision_name    = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
-  mesh_config_name = "istio-${local.revision_name}"
   // CNI should be enabled if either enable_cni or enable_mdp are set
   enable_cni = var.enable_cni || var.enable_mdp
 }
 
-data "google_container_cluster" "asm_cluster" {
+data "google_container_cluster" "asm" {
   project  = var.project_id
   name     = var.cluster_name
   location = var.cluster_location
 }
 
-resource "kubernetes_namespace" "system_namespace" {
+resource "kubernetes_namespace" "system" {
   metadata {
     name = "istio-system"
   }
@@ -42,8 +40,8 @@ resource "kubernetes_config_map" "mesh_config" {
   count = length(var.mesh_config) == 0 ? 0 : 1
 
   metadata {
-    name      = local.mesh_config_name
-    namespace = kubernetes_namespace.system_namespace.metadata[0].name
+    name      = "istio-${local.revision_name}"
+    namespace = kubernetes_namespace.system.metadata[0].name
     labels = {
       "istio.io/rev" = local.revision_name
     }
@@ -57,7 +55,7 @@ resource "kubernetes_config_map" "mesh_config" {
 resource "kubernetes_config_map" "asm_options" {
   metadata {
     name      = "asm-options"
-    namespace = kubernetes_namespace.system_namespace.metadata[0].name
+    namespace = kubernetes_namespace.system.metadata[0].name
 }
 
 
diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index 5bd7163f34..cc9b7c3434 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -21,7 +21,12 @@ if [ "$#" -lt 3 ]; then
     exit 1
 fi
 
-# Wait for the CRD to get created before creating the CPR.
+REVISION_NAME=$1; shift
+CHANNEL=$1; shift
+ENABLE_CNI=$1; shift
+
+# Wait for the CRD to get created before creating the CPR. Not possible to use `kubectl --wait ...` here since
+# the CRD won't exist at the time of checking (https://stackoverflow.com/questions/57115602/how-to-kubectl-wait-for-crd-creation)
 readonly CPR_RESOURCE=controlplanerevisions.mesh.cloud.google.com
 for _i in {1..18}; do
   echo "Ensuring ControlPlaneRevision exists in cluster... attempt ${_i}"
@@ -35,10 +40,6 @@ done
 
 kubectl wait --for condition=established --timeout=60s crd/"${CPR_RESOURCE}"
 
-REVISION_NAME=$1; shift
-CHANNEL=$1; shift
-ENABLE_CNI=$1; shift
-
 cat <<EOF | kubectl apply -f -
 apiVersion: mesh.cloud.google.com/v1beta1
 kind: ControlPlaneRevision

From 22dbbea81fbd6686452c97c9b5b5812c1375dee2 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Fri, 11 Feb 2022 17:42:28 -0800
Subject: [PATCH 18/42] Remove meshConfig from module

---
 modules/asm/README.md    |  1 -
 modules/asm/main.tf      | 21 ++-------------------
 modules/asm/variables.tf |  6 ------
 3 files changed, 2 insertions(+), 26 deletions(-)

diff --git a/modules/asm/README.md b/modules/asm/README.md
index 2e7564915c..3183066bbd 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -15,7 +15,6 @@ This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh
 | enable\_cni | Determines whether to enable CNI for this ASM installation. | `bool` | `false` | no |
 | enable\_cross\_cluster\_service\_discovery | Determines whether to enable cross-cluster service discovery between this cluster and other clusters in the fleet. | `bool` | `false` | no |
 | enable\_mdp | Determines whether to enable Managed Data Plane (MDP) for this ASM installation. | `bool` | `false` | no |
-| mesh\_config | MeshConfig specifies configuration available to the control plane. If unset the module will not attempt to create the MeshConfig (i.e. if managing this configuration elsewhere). The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig | `map(any)` | `{}` | no |
 | project\_id | The project in which the resource belongs. | `string` | n/a | yes |
 
 ## Outputs
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index cfc38f2b4c..f9c42db274 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -36,28 +36,11 @@ resource "kubernetes_namespace" "system" {
   }
 }
 
-resource "kubernetes_config_map" "mesh_config" {
-  count = length(var.mesh_config) == 0 ? 0 : 1
-
-  metadata {
-    name      = "istio-${local.revision_name}"
-    namespace = kubernetes_namespace.system.metadata[0].name
-    labels = {
-      "istio.io/rev" = local.revision_name
-    }
-  }
-
-  data = {
-    mesh = yamlencode(var.mesh_config)
-  }
-}
-
 resource "kubernetes_config_map" "asm_options" {
   metadata {
     name      = "asm-options"
     namespace = kubernetes_namespace.system.metadata[0].name
-}
-
+  }
 
   data = {
     CROSS_CLUSTER_SERVICE_DISCOVERY = var.enable_cross_cluster_service_discovery ? "ON" : "OFF"
@@ -83,5 +66,5 @@ module "cpr" {
   kubectl_create_command  = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${local.enable_cni}"
   kubectl_destroy_command = "${path.module}/scripts/destroy_cpr.sh ${local.revision_name}"
 
-  module_depends_on = [kubernetes_config_map.mesh_config, kubernetes_config_map.asm_options]
+  module_depends_on = [kubernetes_config_map.asm_options]
 }
diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf
index 76a8721fdd..fff476c2c8 100644
--- a/modules/asm/variables.tf
+++ b/modules/asm/variables.tf
@@ -63,9 +63,3 @@ variable "enable_cross_cluster_service_discovery" {
   type        = bool
   default     = false
 }
-
-variable "mesh_config" {
-  description = "MeshConfig specifies configuration available to the control plane. If unset the module will not attempt to create the MeshConfig (i.e. if managing this configuration elsewhere). The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig"
-  type        = map(any)
-  default     = {}
-}

From 3f0e6b2d78e2a5153473a4dc1580cb38ac226a02 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Mon, 14 Feb 2022 17:12:45 -0800
Subject: [PATCH 19/42] fix end to end tests

---
 examples/simple_zonal_with_asm/hub.tf               |  2 +-
 .../simple_zonal_with_asm/controls/kubectl.rb       | 13 -------------
 2 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/examples/simple_zonal_with_asm/hub.tf b/examples/simple_zonal_with_asm/hub.tf
index 036f360adf..d5f6a8f72a 100644
--- a/examples/simple_zonal_with_asm/hub.tf
+++ b/examples/simple_zonal_with_asm/hub.tf
@@ -24,7 +24,7 @@ resource "google_gke_hub_feature" "mesh_feature" {
 resource "google_gke_hub_membership" "cluster_membership" {
   provider      = google-beta
   project       = var.project_id
-  membership_id = "${module.gke.name}-membership"
+  membership_id = "gke-asm-membership"
   endpoint {
     gke_cluster {
       resource_link = "//container.googleapis.com/${module.gke.cluster_id}"
diff --git a/test/integration/simple_zonal_with_asm/controls/kubectl.rb b/test/integration/simple_zonal_with_asm/controls/kubectl.rb
index e0b42a317f..0cc33e1fa0 100644
--- a/test/integration/simple_zonal_with_asm/controls/kubectl.rb
+++ b/test/integration/simple_zonal_with_asm/controls/kubectl.rb
@@ -41,18 +41,5 @@
         },
       )
     end
-
-    describe "Mesh" do
-      describe "CA" do
-        let(:pod) { client.get_pods(label_selector:"app=istio-ingressgateway", namespace: "istio-system", as: :raw) }
-        it "ingressgateway exists" do
-          expect(pod).not_to be_nil
-        end
-
-        it "ingressgateway has correct CA_ADDR " do
-          expect(pod).to include("{\"name\":\"CA_ADDR\",\"value\":\"meshca.googleapis.com:443\"}")
-        end
-      end
-    end
   end
 end

From 83a1112d94c3e4cdc9f74cf5edb6b718a638b1c2 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 15 Feb 2022 09:26:36 -0800
Subject: [PATCH 20/42] lint fixes

---
 examples/simple_zonal_with_asm/main.tf | 10 +++++-----
 modules/asm/main.tf                    |  6 +++---
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf
index 61bbeae946..452b8c5692 100644
--- a/examples/simple_zonal_with_asm/main.tf
+++ b/examples/simple_zonal_with_asm/main.tf
@@ -57,10 +57,10 @@ module "gke" {
 }
 
 module "asm" {
-  source           = "../../modules/asm"
-  project_id       = var.project_id
-  cluster_name     = module.gke.name
-  cluster_location = module.gke.location
+  source                                 = "../../modules/asm"
+  project_id                             = var.project_id
+  cluster_name                           = module.gke.name
+  cluster_location                       = module.gke.location
   enable_cross_cluster_service_discovery = true
-  enable_cni = true
+  enable_cni                             = true
 }
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index f9c42db274..74be6e5fda 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -16,10 +16,10 @@
 
 locals {
   // GKE release channel is a list with max length 1 https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
-  gke_release_channel       = data.google_container_cluster.asm.release_channel != null ? data.google_container_cluster.asm.release_channel[0].channel : ""
+  gke_release_channel = data.google_container_cluster.asm.release_channel != null ? data.google_container_cluster.asm.release_channel[0].channel : ""
   // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
-  channel          = lower(coalesce(var.channel, local.gke_release_channel, "regular"))
-  revision_name    = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
+  channel       = lower(coalesce(var.channel, local.gke_release_channel, "regular"))
+  revision_name = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
   // CNI should be enabled if either enable_cni or enable_mdp are set
   enable_cni = var.enable_cni || var.enable_mdp
 }

From a7879406a3f58ba944957ef090ad13a6100da48e Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 15 Feb 2022 10:51:15 -0800
Subject: [PATCH 21/42] Remove enable_mdp, add enable_vpc_sc and fleet_id

---
 examples/simple_zonal_with_asm/hub.tf |  1 -
 modules/asm/main.tf                   |  6 +++---
 modules/asm/scripts/create_cpr.sh     |  2 ++
 modules/asm/variables.tf              | 14 +++++++++-----
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/examples/simple_zonal_with_asm/hub.tf b/examples/simple_zonal_with_asm/hub.tf
index d5f6a8f72a..1cb8221dbb 100644
--- a/examples/simple_zonal_with_asm/hub.tf
+++ b/examples/simple_zonal_with_asm/hub.tf
@@ -31,4 +31,3 @@ resource "google_gke_hub_membership" "cluster_membership" {
     }
   }
 }
-
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index 74be6e5fda..d585ac6031 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -20,8 +20,8 @@ locals {
   // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
   channel       = lower(coalesce(var.channel, local.gke_release_channel, "regular"))
   revision_name = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
-  // CNI should be enabled if either enable_cni or enable_mdp are set
-  enable_cni = var.enable_cni || var.enable_mdp
+  // Fleet ID should default to project ID if unset
+  fleet_id = coalesce(var.fleet_id, var.project_id)
 }
 
 data "google_container_cluster" "asm" {
@@ -63,7 +63,7 @@ module "cpr" {
   cluster_name     = var.cluster_name
   cluster_location = var.cluster_location
 
-  kubectl_create_command  = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${local.enable_cni}"
+  kubectl_create_command  = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${var.enable_cni} ${var.enable_vpc_sc}"
   kubectl_destroy_command = "${path.module}/scripts/destroy_cpr.sh ${local.revision_name}"
 
   module_depends_on = [kubernetes_config_map.asm_options]
diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index cc9b7c3434..49110a2727 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -24,6 +24,7 @@ fi
 REVISION_NAME=$1; shift
 CHANNEL=$1; shift
 ENABLE_CNI=$1; shift
+ENABLE_VPC_SC=$1; shift
 
 # Wait for the CRD to get created before creating the CPR. Not possible to use `kubectl --wait ...` here since
 # the CRD won't exist at the time of checking (https://stackoverflow.com/questions/57115602/how-to-kubectl-wait-for-crd-creation)
@@ -48,6 +49,7 @@ metadata:
   namespace: istio-system
   labels:
     mesh.cloud.google.com/managed-cni-enabled: "${ENABLE_CNI}"
+    mesh.cloud.google.com/vpcsc: "${ENABLE_VPC_SC}"
 spec:
   type: managed_service
   channel: "${CHANNEL}"
diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf
index fff476c2c8..88bfd1c12f 100644
--- a/modules/asm/variables.tf
+++ b/modules/asm/variables.tf
@@ -29,6 +29,12 @@ variable "cluster_location" {
   type        = string
 }
 
+variable "fleet_id" {
+  description = "The fleet to use for this ASM installation."
+  type        = string
+  default     = ""
+}
+
 variable "channel" {
   description = "The channel to use for this ASM installation."
   type        = string
@@ -45,15 +51,13 @@ variable "channel" {
 }
 
 variable "enable_cni" {
-  description = "Determines whether to enable CNI for this ASM installation."
+  description = "Determines whether to enable CNI for this ASM installation. Required to use Managed Data Plane (MDP)."
   type        = bool
   default     = false
 }
 
-// This should be validated so that it cannot be enabled while CNI is disabled
-// but validating based on other variables is not possible today (https://github.com/hashicorp/terraform/issues/25609)
-variable "enable_mdp" {
-  description = "Determines whether to enable Managed Data Plane (MDP) for this ASM installation."
+variable "enable_vpc_sc" {
+  description = "Determines whether to enable VPC-SC for this ASM installation. For more information read https://cloud.google.com/service-mesh/docs/managed/vpc-sc"
   type        = bool
   default     = false
 }

From 3be283c818f8d5b2333ac4da7a1fec25aadc157e Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 15 Feb 2022 10:57:56 -0800
Subject: [PATCH 22/42] move VPC-SC from labels to annotations

---
 modules/asm/scripts/create_cpr.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index 49110a2727..849993068c 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -47,9 +47,10 @@ kind: ControlPlaneRevision
 metadata:
   name: "${REVISION_NAME}"
   namespace: istio-system
+  annotations:
+    mesh.cloud.google.com/vpcsc: "${ENABLE_VPC_SC}"
   labels:
     mesh.cloud.google.com/managed-cni-enabled: "${ENABLE_CNI}"
-    mesh.cloud.google.com/vpcsc: "${ENABLE_VPC_SC}"
 spec:
   type: managed_service
   channel: "${CHANNEL}"

From 5af57bf73bc48fdde0242177344d44224bebb825 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 15 Feb 2022 11:00:56 -0800
Subject: [PATCH 23/42] update README

---
 modules/asm/README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/modules/asm/README.md b/modules/asm/README.md
index 3183066bbd..cc795afbb0 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -12,9 +12,10 @@ This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh
 | channel | The channel to use for this ASM installation. | `string` | `""` | no |
 | cluster\_location | The cluster location for this ASM installation. | `string` | n/a | yes |
 | cluster\_name | The unique name to identify the cluster in ASM. | `string` | n/a | yes |
-| enable\_cni | Determines whether to enable CNI for this ASM installation. | `bool` | `false` | no |
+| enable\_cni | Determines whether to enable CNI for this ASM installation. Required to use Managed Data Plane (MDP). | `bool` | `false` | no |
 | enable\_cross\_cluster\_service\_discovery | Determines whether to enable cross-cluster service discovery between this cluster and other clusters in the fleet. | `bool` | `false` | no |
-| enable\_mdp | Determines whether to enable Managed Data Plane (MDP) for this ASM installation. | `bool` | `false` | no |
+| enable\_vpc\_sc | Determines whether to enable VPC-SC for this ASM installation. For more information read https://cloud.google.com/service-mesh/docs/managed/vpc-sc | `bool` | `false` | no |
+| fleet\_id | The fleet to use for this ASM installation. | `string` | `""` | no |
 | project\_id | The project in which the resource belongs. | `string` | n/a | yes |
 
 ## Outputs

From 347c75a1a189fbb7e8e8897dde87cc286e817fdf Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 15 Feb 2022 11:45:43 -0800
Subject: [PATCH 24/42] use default node pool size

---
 examples/simple_zonal_with_asm/main.tf | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf
index 452b8c5692..ac0630bec9 100644
--- a/examples/simple_zonal_with_asm/main.tf
+++ b/examples/simple_zonal_with_asm/main.tf
@@ -49,8 +49,7 @@ module "gke" {
       name         = "asm-node-pool"
       autoscaling  = false
       auto_upgrade = true
-      # ASM requires minimum 4 nodes and e2-standard-4
-      node_count   = 4
+      node_count   = 3
       machine_type = "e2-standard-4"
     },
   ]

From 1bc68e48894211a707e0b04c37420a42c211b02c Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 15 Feb 2022 11:50:16 -0800
Subject: [PATCH 25/42] use wip for exmaple cluster creation

---
 examples/simple_zonal_with_asm/main.tf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf
index ac0630bec9..2b7805fa18 100644
--- a/examples/simple_zonal_with_asm/main.tf
+++ b/examples/simple_zonal_with_asm/main.tf
@@ -44,6 +44,7 @@ module "gke" {
   ip_range_services       = var.ip_range_services
   network_policy          = false
   cluster_resource_labels = { "mesh_id" : "proj-${data.google_project.project.number}" }
+  identity_namespace = "${var.project_id}.svc.id.goog"
   node_pools = [
     {
       name         = "asm-node-pool"

From 3801f7bd520febcf2fade2ea2831d441bca3c4e5 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 15 Feb 2022 14:44:39 -0800
Subject: [PATCH 26/42] add feature enablement back to module

---
 examples/simple_zonal_with_asm/hub.tf |  7 -------
 modules/asm/main.tf                   | 18 +++++++++---------
 2 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/examples/simple_zonal_with_asm/hub.tf b/examples/simple_zonal_with_asm/hub.tf
index 1cb8221dbb..d56fa25dfd 100644
--- a/examples/simple_zonal_with_asm/hub.tf
+++ b/examples/simple_zonal_with_asm/hub.tf
@@ -14,13 +14,6 @@
  * limitations under the License.
  */
 
-resource "google_gke_hub_feature" "mesh_feature" {
-  name     = "servicemesh"
-  project  = var.project_id
-  location = "global"
-  provider = google-beta
-}
-
 resource "google_gke_hub_membership" "cluster_membership" {
   provider      = google-beta
   project       = var.project_id
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index d585ac6031..e401e542c7 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -30,6 +30,13 @@ data "google_container_cluster" "asm" {
   location = var.cluster_location
 }
 
+resource "google_gke_hub_feature" "mesh" {
+  name     = "servicemesh"
+  project  = var.project_id
+  location = "global"
+  provider = google-beta
+}
+
 resource "kubernetes_namespace" "system" {
   metadata {
     name = "istio-system"
@@ -45,16 +52,9 @@ resource "kubernetes_config_map" "asm_options" {
   data = {
     CROSS_CLUSTER_SERVICE_DISCOVERY = var.enable_cross_cluster_service_discovery ? "ON" : "OFF"
   }
-}
 
-# TODO(Monkeyanator) due to a bug in the gke_hub_feature resource implementation this fails when enabling an enabled
-# feature (i.e. running the module twice) and does not disable the feature upon TF destroy.
-#resource "google_gke_hub_feature" "mesh_feature" {
-#  name     = "servicemesh"
-#  project  = var.project_id
-#  location = "global"
-#  provider = google-beta
-#}
+  depends_on = [google_gke_hub_feature.mesh]
+}
 
 module "cpr" {
   source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"

From 9e63a53a4a290ead9b5a34392d7376381f5e5995 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 15 Feb 2022 14:52:29 -0800
Subject: [PATCH 27/42] lint

---
 examples/simple_zonal_with_asm/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf
index 2b7805fa18..224750f5a8 100644
--- a/examples/simple_zonal_with_asm/main.tf
+++ b/examples/simple_zonal_with_asm/main.tf
@@ -44,7 +44,7 @@ module "gke" {
   ip_range_services       = var.ip_range_services
   network_policy          = false
   cluster_resource_labels = { "mesh_id" : "proj-${data.google_project.project.number}" }
-  identity_namespace = "${var.project_id}.svc.id.goog"
+  identity_namespace      = "${var.project_id}.svc.id.goog"
   node_pools = [
     {
       name         = "asm-node-pool"

From c963e4c4edf435c237f8558d616732ff29f61fcd Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Wed, 16 Feb 2022 10:12:31 -0800
Subject: [PATCH 28/42] remove feature enablement from module

---
 examples/simple_zonal_with_asm/hub.tf |  7 +++++++
 modules/asm/main.tf                   | 12 ++++++------
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/examples/simple_zonal_with_asm/hub.tf b/examples/simple_zonal_with_asm/hub.tf
index d56fa25dfd..8fe8b1f6d9 100644
--- a/examples/simple_zonal_with_asm/hub.tf
+++ b/examples/simple_zonal_with_asm/hub.tf
@@ -24,3 +24,10 @@ resource "google_gke_hub_membership" "cluster_membership" {
     }
   }
 }
+
+resource "google_gke_hub_feature" "mesh" {
+  name     = "servicemesh"
+  project  = var.project_id
+  location = "global"
+  provider = google-beta
+}
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index e401e542c7..ab01997d57 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -30,12 +30,12 @@ data "google_container_cluster" "asm" {
   location = var.cluster_location
 }
 
-resource "google_gke_hub_feature" "mesh" {
-  name     = "servicemesh"
-  project  = var.project_id
-  location = "global"
-  provider = google-beta
-}
+#resource "google_gke_hub_feature" "mesh" {
+#  name     = "servicemesh"
+#  project  = var.project_id
+#  location = "global"
+#  provider = google-beta
+#}
 
 resource "kubernetes_namespace" "system" {
   metadata {

From aea6fe544e563c02e0eef638b1146ad24f1c382d Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Wed, 16 Feb 2022 10:17:14 -0800
Subject: [PATCH 29/42] remove depends_on

---
 modules/asm/main.tf | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index ab01997d57..acf11a395c 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -52,8 +52,6 @@ resource "kubernetes_config_map" "asm_options" {
   data = {
     CROSS_CLUSTER_SERVICE_DISCOVERY = var.enable_cross_cluster_service_discovery ? "ON" : "OFF"
   }
-
-  depends_on = [google_gke_hub_feature.mesh]
 }
 
 module "cpr" {

From 30ea4ba3b1611aee3aeb2fe2fec2155a63bd2ee9 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Wed, 23 Feb 2022 13:10:18 -0800
Subject: [PATCH 30/42] fix unspecified channel bug

---
 modules/asm/main.tf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index acf11a395c..562ea5fb41 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -17,8 +17,9 @@
 locals {
   // GKE release channel is a list with max length 1 https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
   gke_release_channel = data.google_container_cluster.asm.release_channel != null ? data.google_container_cluster.asm.release_channel[0].channel : ""
+  gke_release_channel_filtered = lower(local.gke_release_channel) == "unspecified" ? "" : local.gke_release_channel
   // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
-  channel       = lower(coalesce(var.channel, local.gke_release_channel, "regular"))
+  channel       = lower(coalesce(var.channel, local.gke_release_channel_filtered, "regular"))
   revision_name = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
   // Fleet ID should default to project ID if unset
   fleet_id = coalesce(var.fleet_id, var.project_id)

From 3ae8025db44527961cf6461a09cd6bf36436c001 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Wed, 23 Feb 2022 13:45:24 -0800
Subject: [PATCH 31/42] minor fixes

---
 modules/asm/main.tf    | 5 +++--
 modules/asm/outputs.tf | 5 +++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index 562ea5fb41..b8b8433523 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -16,7 +16,7 @@
 
 locals {
   // GKE release channel is a list with max length 1 https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
-  gke_release_channel = data.google_container_cluster.asm.release_channel != null ? data.google_container_cluster.asm.release_channel[0].channel : ""
+  gke_release_channel          = data.google_container_cluster.asm.release_channel != null ? data.google_container_cluster.asm.release_channel[0].channel : ""
   gke_release_channel_filtered = lower(local.gke_release_channel) == "unspecified" ? "" : local.gke_release_channel
   // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
   channel       = lower(coalesce(var.channel, local.gke_release_channel_filtered, "regular"))
@@ -56,7 +56,8 @@ resource "kubernetes_config_map" "asm_options" {
 }
 
 module "cpr" {
-  source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+  version = "3.1.0"
 
   project_id       = var.project_id
   cluster_name     = var.cluster_name
diff --git a/modules/asm/outputs.tf b/modules/asm/outputs.tf
index 4a82664e70..ba9237238c 100644
--- a/modules/asm/outputs.tf
+++ b/modules/asm/outputs.tf
@@ -18,3 +18,8 @@ output "revision_name" {
   value       = local.revision_name
   description = "The name of the installed managed ASM revision."
 }
+
+output "wait" {
+  value       = module.cpr.wait
+  description = "An output to use when depending on the ASM installation finishing."
+}

From 7a4867ef188862a2a6e22f5105a4f0160a083909 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Wed, 23 Feb 2022 15:29:20 -0800
Subject: [PATCH 32/42] add more testing

---
 modules/asm/main.tf                           |  2 +-
 .../simple_zonal_with_asm/controls/kubectl.rb | 21 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index b8b8433523..4ba205a8a3 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -51,7 +51,7 @@ resource "kubernetes_config_map" "asm_options" {
   }
 
   data = {
-    CROSS_CLUSTER_SERVICE_DISCOVERY = var.enable_cross_cluster_service_discovery ? "ON" : "OFF"
+    CROSS_CLUSTER_SERVICE_DISCOVERY = var.enable_cross_cluster_service_discovery ? "on" : "off"
   }
 }
 
diff --git a/test/integration/simple_zonal_with_asm/controls/kubectl.rb b/test/integration/simple_zonal_with_asm/controls/kubectl.rb
index 0cc33e1fa0..71a89f5b07 100644
--- a/test/integration/simple_zonal_with_asm/controls/kubectl.rb
+++ b/test/integration/simple_zonal_with_asm/controls/kubectl.rb
@@ -41,5 +41,26 @@
         },
       )
     end
+
+    describe "configmap" do
+      describe "asm-options" do
+        let(:asmoptions_configmap) { client.get_config_map("asm-options", "istio-system") }
+
+        it "exists" do
+          expect(asmoptions_configmap.metadata.name).to eq "asm-options"
+        end
+
+        it "has multicluster disabled" do
+          expect(asmoptions_configmap.data.CROSS_CLUSTER_SERVICE_DISCOVERY).to eq "off"
+        end
+      end
+    end
+
+    describe "namespace" do
+      let(:system_namespace) { client.get_namespace("istio-system") }
+      it "exists" do
+        expect(system_namespace).not_to be_nil
+      end
+    end
   end
 end

From f92c91e8f31e598f558a3406b6a75ccd0f38fea4 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Wed, 23 Feb 2022 15:37:20 -0800
Subject: [PATCH 33/42] update docs

---
 modules/asm/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/asm/README.md b/modules/asm/README.md
index cc795afbb0..874eaf6b06 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -23,5 +23,6 @@ This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh
 | Name | Description |
 |------|-------------|
 | revision\_name | The name of the installed managed ASM revision. |
+| wait | An output to use when depending on the ASM installation finishing. |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

From 33c742dc2df38250a54d68d79448bddfafd9f304 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Wed, 23 Feb 2022 15:46:57 -0800
Subject: [PATCH 34/42] change required module version

---
 modules/asm/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index 4ba205a8a3..a5d532beb7 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -57,7 +57,7 @@ resource "kubernetes_config_map" "asm_options" {
 
 module "cpr" {
   source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
-  version = "3.1.0"
+  version = "~> 3.1"
 
   project_id       = var.project_id
   cluster_name     = var.cluster_name

From 036ecc8e1f3a6d0220f31f0237042bc56dadc403 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Wed, 23 Feb 2022 16:04:08 -0800
Subject: [PATCH 35/42] fix cclb

---
 test/integration/simple_zonal_with_asm/controls/kubectl.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/simple_zonal_with_asm/controls/kubectl.rb b/test/integration/simple_zonal_with_asm/controls/kubectl.rb
index 71a89f5b07..ca45c929f3 100644
--- a/test/integration/simple_zonal_with_asm/controls/kubectl.rb
+++ b/test/integration/simple_zonal_with_asm/controls/kubectl.rb
@@ -51,7 +51,7 @@
         end
 
         it "has multicluster disabled" do
-          expect(asmoptions_configmap.data.CROSS_CLUSTER_SERVICE_DISCOVERY).to eq "off"
+          expect(asmoptions_configmap.data.CROSS_CLUSTER_SERVICE_DISCOVERY).to eq "on"
         end
       end
     end

From 04484eeae10f4dcdf9c3afb62c65fa8fca3aba40 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Thu, 24 Feb 2022 09:00:34 -0800
Subject: [PATCH 36/42] change registry source to run in CI

---
 test/setup/main.tf | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/setup/main.tf b/test/setup/main.tf
index a4bee2d789..e3015cbda8 100644
--- a/test/setup/main.tf
+++ b/test/setup/main.tf
@@ -75,10 +75,8 @@ module "gke-project-2" {
 
 # apis as documented https://cloud.google.com/service-mesh/docs/scripted-install/reference#setting_up_your_project
 module "gke-project-asm" {
-  source = "github.com/terraform-google-modules/terraform-google-project-factory.git?ref=master"
-
-  #source  = "terraform-google-modules/project-factory/google"
-  #version = "~> 11.3"
+  source  = "terraform-google-modules/project-factory/google"
+  version = "~> 11.3"
 
   name              = "ci-gke-asm-${random_id.random_project_id_suffix.hex}"
   random_project_id = true

From d46465a6677ff3b8512fe13c5c15d60c501c5a8e Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Thu, 24 Feb 2022 12:26:22 -0800
Subject: [PATCH 37/42] update CROSS_CLUSTER_SERVICE_DISCOVERY to
 multicluster_mode

---
 examples/simple_zonal_with_asm/main.tf | 12 ++++++------
 modules/asm/README.md                  |  2 +-
 modules/asm/main.tf                    |  2 +-
 modules/asm/variables.tf               | 19 +++++++++++++------
 4 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf
index 224750f5a8..871f81670f 100644
--- a/examples/simple_zonal_with_asm/main.tf
+++ b/examples/simple_zonal_with_asm/main.tf
@@ -57,10 +57,10 @@ module "gke" {
 }
 
 module "asm" {
-  source                                 = "../../modules/asm"
-  project_id                             = var.project_id
-  cluster_name                           = module.gke.name
-  cluster_location                       = module.gke.location
-  enable_cross_cluster_service_discovery = true
-  enable_cni                             = true
+  source            = "../../modules/asm"
+  project_id        = var.project_id
+  cluster_name      = module.gke.name
+  cluster_location  = module.gke.location
+  multicluster_mode = "connected"
+  enable_cni        = true
 }
diff --git a/modules/asm/README.md b/modules/asm/README.md
index 874eaf6b06..319bcd53d1 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -13,9 +13,9 @@ This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh
 | cluster\_location | The cluster location for this ASM installation. | `string` | n/a | yes |
 | cluster\_name | The unique name to identify the cluster in ASM. | `string` | n/a | yes |
 | enable\_cni | Determines whether to enable CNI for this ASM installation. Required to use Managed Data Plane (MDP). | `bool` | `false` | no |
-| enable\_cross\_cluster\_service\_discovery | Determines whether to enable cross-cluster service discovery between this cluster and other clusters in the fleet. | `bool` | `false` | no |
 | enable\_vpc\_sc | Determines whether to enable VPC-SC for this ASM installation. For more information read https://cloud.google.com/service-mesh/docs/managed/vpc-sc | `bool` | `false` | no |
 | fleet\_id | The fleet to use for this ASM installation. | `string` | `""` | no |
+| multicluster\_mode | [Preview] Determines whether remote secrets should be autogenerated across fleet cluster. | `string` | `"manual"` | no |
 | project\_id | The project in which the resource belongs. | `string` | n/a | yes |
 
 ## Outputs
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index a5d532beb7..44298241a5 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -51,7 +51,7 @@ resource "kubernetes_config_map" "asm_options" {
   }
 
   data = {
-    CROSS_CLUSTER_SERVICE_DISCOVERY = var.enable_cross_cluster_service_discovery ? "on" : "off"
+    multicluster_mode = var.multicluster_mode
   }
 }
 
diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf
index 88bfd1c12f..1cda2eb52e 100644
--- a/modules/asm/variables.tf
+++ b/modules/asm/variables.tf
@@ -50,6 +50,19 @@ variable "channel" {
   default = ""
 }
 
+variable "multicluster_mode" {
+  description = "[Preview] Determines whether remote secrets should be autogenerated across fleet cluster."
+  type        = string
+  validation {
+    condition = anytrue([
+      var.channel == "manual",
+      var.channel == "connected",
+    ])
+    error_message = "Must be one of manual or connected."
+  }
+  default = "manual"
+}
+
 variable "enable_cni" {
   description = "Determines whether to enable CNI for this ASM installation. Required to use Managed Data Plane (MDP)."
   type        = bool
@@ -61,9 +74,3 @@ variable "enable_vpc_sc" {
   type        = bool
   default     = false
 }
-
-variable "enable_cross_cluster_service_discovery" {
-  description = "Determines whether to enable cross-cluster service discovery between this cluster and other clusters in the fleet."
-  type        = bool
-  default     = false
-}

From 3848a0dc9cc20e4525df00b0f397cd9b93de933a Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Thu, 24 Feb 2022 16:28:35 -0800
Subject: [PATCH 38/42] update README

---
 modules/asm/README.md    | 10 ++++++++++
 modules/asm/variables.tf |  4 ++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/modules/asm/README.md b/modules/asm/README.md
index 319bcd53d1..ba75476125 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -4,6 +4,16 @@ This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh
 
 ## Usage
 
+There are a few prerequisites to using this module that can be done either through Terraform or manually:
+
+1. Enable the `mesh.cloud.googleapis.com` service
+1. Enable the `servicemesh` feature on the cluster hub
+1. Register target cluster to the servicemesh-enabled hub
+
+## Migration
+
+Direct migration from the previous Terraform module is not supported in the first iteration of this module.
+
  <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf
index 1cda2eb52e..71efd3c59e 100644
--- a/modules/asm/variables.tf
+++ b/modules/asm/variables.tf
@@ -55,8 +55,8 @@ variable "multicluster_mode" {
   type        = string
   validation {
     condition = anytrue([
-      var.channel == "manual",
-      var.channel == "connected",
+      var.multicluster_mode == "manual",
+      var.multicluster_mode == "connected",
     ])
     error_message = "Must be one of manual or connected."
   }

From d1959812e0ae2ec84d7656b3293c0324a11566f7 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 1 Mar 2022 13:52:43 -0800
Subject: [PATCH 39/42] fix test

---
 test/integration/simple_zonal_with_asm/controls/kubectl.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/integration/simple_zonal_with_asm/controls/kubectl.rb b/test/integration/simple_zonal_with_asm/controls/kubectl.rb
index ca45c929f3..c42f8554ea 100644
--- a/test/integration/simple_zonal_with_asm/controls/kubectl.rb
+++ b/test/integration/simple_zonal_with_asm/controls/kubectl.rb
@@ -50,8 +50,8 @@
           expect(asmoptions_configmap.metadata.name).to eq "asm-options"
         end
 
-        it "has multicluster disabled" do
-          expect(asmoptions_configmap.data.CROSS_CLUSTER_SERVICE_DISCOVERY).to eq "on"
+        it "has multicluster_mode connected" do
+          expect(asmoptions_configmap.data.multicluster_mode).to eq "connected"
         end
       end
     end

From 6faadcf8960d442a37dbe78753e9990189582ef9 Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 1 Mar 2022 14:22:16 -0800
Subject: [PATCH 40/42] fix wording

---
 modules/asm/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/asm/README.md b/modules/asm/README.md
index ba75476125..ae0d77e0ce 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -12,7 +12,7 @@ There are a few prerequisites to using this module that can be done either throu
 
 ## Migration
 
-Direct migration from the previous Terraform module is not supported in the first iteration of this module.
+Direct migration from the previous Terraform module implementation is not supported in the first iteration of this module.
 
  <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs

From a8e32f5cb3fc9a72d4aacb45ec194a4de41b778e Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Tue, 1 Mar 2022 15:41:11 -0800
Subject: [PATCH 41/42] remove from README

---
 modules/asm/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/asm/README.md b/modules/asm/README.md
index ae0d77e0ce..ba75476125 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -12,7 +12,7 @@ There are a few prerequisites to using this module that can be done either throu
 
 ## Migration
 
-Direct migration from the previous Terraform module implementation is not supported in the first iteration of this module.
+Direct migration from the previous Terraform module is not supported in the first iteration of this module.
 
  <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs

From bc7449d32316f416890ed3614aebb3ae21b979bf Mon Sep 17 00:00:00 2001
From: Sam Naser <samnaser@google.com>
Date: Wed, 2 Mar 2022 14:14:05 -0800
Subject: [PATCH 42/42] iterate on comments

---
 docs/upgrading_to_v20.0.md                       | 10 ++++++++++
 modules/asm/README.md                            | 16 ++++++++++++++--
 modules/asm/main.tf                              |  7 -------
 modules/asm/scripts/create_cpr.sh                |  2 +-
 .../simple_zonal_with_asm/controls/kubectl.rb    |  4 ----
 5 files changed, 25 insertions(+), 14 deletions(-)
 create mode 100644 docs/upgrading_to_v20.0.md

diff --git a/docs/upgrading_to_v20.0.md b/docs/upgrading_to_v20.0.md
new file mode 100644
index 0000000000..6e24483370
--- /dev/null
+++ b/docs/upgrading_to_v20.0.md
@@ -0,0 +1,10 @@
+# Upgrading to v20.0
+
+The v20.0 release of *kubernetes-engine* is a backwards incompatible
+release for the Anthos Service Mesh (ASM) module.
+
+### ASM module rewrite
+
+The [ASM submodule](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/asm) has been rewritten to use the `ControlPlaneRevision` API to provision
+a managed control plane rather than using an installer script. Due to the drastic difference in implementation the module does not support an upgrade path
+from the previous version.
diff --git a/modules/asm/README.md b/modules/asm/README.md
index ba75476125..11be0c1ab1 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -10,9 +10,21 @@ There are a few prerequisites to using this module that can be done either throu
 1. Enable the `servicemesh` feature on the cluster hub
 1. Register target cluster to the servicemesh-enabled hub
 
-## Migration
+There is a full example provided [here](../../examples/simple_zonal_with_asm). Detailed usage example is as follows:
 
-Direct migration from the previous Terraform module is not supported in the first iteration of this module.
+```tf
+module "asm" {
+  source            = "../../modules/asm"
+  project_id        = var.project_id
+  cluster_name      = module.gke.name
+  cluster_location  = module.gke.location
+  enable_cni        = true
+}
+```
+
+To deploy this config:
+
+1. Run `terraform apply`
 
  <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index 44298241a5..72c8c734ed 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -31,13 +31,6 @@ data "google_container_cluster" "asm" {
   location = var.cluster_location
 }
 
-#resource "google_gke_hub_feature" "mesh" {
-#  name     = "servicemesh"
-#  project  = var.project_id
-#  location = "global"
-#  provider = google-beta
-#}
-
 resource "kubernetes_namespace" "system" {
   metadata {
     name = "istio-system"
diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh
index 849993068c..60ad768cbe 100755
--- a/modules/asm/scripts/create_cpr.sh
+++ b/modules/asm/scripts/create_cpr.sh
@@ -16,7 +16,7 @@
 
 set -e
 
-if [ "$#" -lt 3 ]; then
+if [ "$#" -lt 4 ]; then
     >&2 echo "Not all expected arguments set."
     exit 1
 fi
diff --git a/test/integration/simple_zonal_with_asm/controls/kubectl.rb b/test/integration/simple_zonal_with_asm/controls/kubectl.rb
index c42f8554ea..6d4370de06 100644
--- a/test/integration/simple_zonal_with_asm/controls/kubectl.rb
+++ b/test/integration/simple_zonal_with_asm/controls/kubectl.rb
@@ -49,10 +49,6 @@
         it "exists" do
           expect(asmoptions_configmap.metadata.name).to eq "asm-options"
         end
-
-        it "has multicluster_mode connected" do
-          expect(asmoptions_configmap.data.multicluster_mode).to eq "connected"
-        end
       end
     end