Rewrite ASM module

examples/simple_zonal_with_asm/hub.tf

@@ -14,11 +14,21 @@
  * limitations under the License.
  */
 
-module "hub" {
-  source                  = "../../modules/hub"
-  project_id              = var.project_id
-  location                = module.gke.location
-  cluster_name            = module.gke.name
-  cluster_endpoint        = module.gke.endpoint
-  gke_hub_membership_name = "gke-asm-membership"
+resource "google_gke_hub_feature" "mesh_feature" {
+  name     = "servicemesh"
+  project  = var.project_id
+  location = "global"
+  provider = google-beta
 }
+
+resource "google_gke_hub_membership" "cluster_membership" {
+  provider      = google-beta
+  project       = var.project_id
+  membership_id = "${module.gke.name}-membership"
+  endpoint {
+    gke_cluster {
+      resource_link = "//container.googleapis.com/${module.gke.cluster_id}"
+    }
+  }
+}
+

examples/simple_zonal_with_asm/main.tf

@@ -57,16 +57,10 @@ module "gke" {
 }
 
 module "asm" {
-  source                    = "../../modules/asm"
-  cluster_name              = module.gke.name
-  cluster_endpoint          = module.gke.endpoint
-  project_id                = var.project_id
-  location                  = module.gke.location
-  enable_cluster_roles      = true
-  enable_cluster_labels     = true
-  enable_gcp_apis           = true
-  enable_gcp_components     = true
-  enable_namespace_creation = true
-  options                   = ["envoy-access-log"]
-  outdir                    = "./${module.gke.name}-outdir"
+  source           = "../../modules/asm"
+  project_id       = var.project_id
+  cluster_name     = module.gke.name
+  cluster_location = module.gke.location
+  enable_cross_cluster_service_discovery = true
+  enable_cni = true
 }

modules/asm/README.md

@@ -2,87 +2,26 @@
 
 This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh/docs) (ASM) in a Kubernetes Engine (GKE) cluster.
 
-Specifically, this module automates installing the ASM Istio Operator on your cluster ([installing ASM](https://cloud.google.com/service-mesh/docs/install)).
-
 ## Usage
 
-There is a [full example](../../examples/simple_zonal_with_asm) provided. Detailed usage example is as follows:
-
-```tf
-module "asm" {
-  source                = "terraform-google-modules/kubernetes-engine/google//modules/asm"
-
-  project_id            = "my-project-id"
-  cluster_name          = "my-cluster-name"
-  location              = module.gke.location
-  cluster_endpoint      = module.gke.endpoint
-  enable_all            = false
-  enable_cluster_roles  = true
-  enable_cluster_labels = false
-  enable_gcp_apis       = false
-  enable_gcp_iam_roles  = true
-  enable_gcp_components = true
-  enable_registration   = false
-  managed_control_plane = false
-  options               = ["envoy-access-log,egressgateways"]
-  custom_overlays       = ["./custom_ingress_gateway.yaml"]
-  skip_validation       = true
-  outdir                = "./${module.gke.name}-outdir-${var.asm_version}"
-}
-```
-
-To deploy this config:
-
-1. Run `terraform apply`
-
-## Requirements
-
-- Anthos Service Mesh on GCP no longer requires an active Anthos license. You can use Anthos Service Mesh as a standalone product on GCP (on GKE) or as part of your Anthos subscription for hybrid and multi-cloud architectures.
-- GKE cluster must have minimum four nodes.
-- Minimum machine type is `e2-standard-4`.
-- GKE cluster must be enrolled in a release channel. ASM does not support static version.
-- ASM on a private GKE cluster requires adding a firewall rule to open port 15017 if you want to use [automatic sidecar injection](https://cloud.google.com/service-mesh/docs/proxy-injection).
-- One ASM mesh per Google Cloud project is supported.
-
  <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| asm\_git\_tag | ASM git tag to deploy. This module supports versions `1.8`, `1.9` and `1.10`. You can get the exact `asm_git_tag` by running the command `install_asm --version`. The ASM git tab should be of the form `1.9.3-asm.2+config5`. You can also see all ASM git tags by running `curl https://storage.googleapis.com/csm-artifacts/asm/STABLE_VERSIONS`. You must provide the full and exact git tag. This variable is optional. Leaving it empty (default) will download the latest `install_asm` script for the version provided by the `asm_version` variable. | `string` | `""` | no |
-| asm\_version | ASM version to deploy. This module supports versions `1.8`, `1.9` and `1.10`. Available versions are documented in https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages | `string` | `"1.9"` | no |
-| ca | Sets CA option. Possible values are `meshca` or `citadel`. Additional documentation on Citadel is available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_citadel_as_the_ca. | `string` | `"meshca"` | no |
-| ca\_certs | Sets CA certificate file paths when `ca` is set to `citadel`. These values must be provided when using Citadel as CA. Additional documentation on Citadel is available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_citadel_as_the_ca. | `map(any)` | `{}` | no |
-| cluster\_endpoint | The GKE cluster endpoint. | `string` | n/a | yes |
+| channel | The channel to use for this ASM installation. | `string` | `""` | no |
+| cluster\_location | The cluster location for this ASM installation. | `string` | n/a | yes |
 | cluster\_name | The unique name to identify the cluster in ASM. | `string` | n/a | yes |
-| custom\_overlays | Comma separated list of custom\_overlay file paths. Works with in-cluster control plane only. Additional documentation available at https://cloud.google.com/service-mesh/docs/scripted-install/gke-install#installation_with_an_overlay_file | `list(any)` | `[]` | no |
-| enable\_all | Sets `--enable_all` option if true. | `bool` | `false` | no |
-| enable\_cluster\_labels | Sets `--enable_cluster_labels` option if true. | `bool` | `false` | no |
-| enable\_cluster\_roles | Sets `--enable_cluster_roles` option if true. | `bool` | `false` | no |
-| enable\_gcp\_apis | Sets `--enable_gcp_apis` option if true. | `bool` | `false` | no |
-| enable\_gcp\_components | Sets --enable\_gcp\_components option if true. Can be true or false. Available versions are documented in https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages | `bool` | `false` | no |
-| enable\_gcp\_iam\_roles | Grants IAM roles required for ASM if true. If enable\_gcp\_iam\_roles, one of impersonate\_service\_account, service\_account, or iam\_member must be set. | `bool` | `false` | no |
-| enable\_namespace\_creation | Sets `--enable_namespace_creation` option if true. | `bool` | `false` | no |
-| enable\_registration | Sets `--enable_registration` option if true. | `bool` | `false` | no |
-| gcloud\_sdk\_version | The gcloud sdk version to use. Minimum required version is 293.0.0 | `string` | `"296.0.1"` | no |
-| iam\_member | The GCP member email address to grant IAM roles to. If impersonate\_service\_account or service\_account is set, roles are granted to that SA. | `string` | `""` | no |
-| impersonate\_service\_account | An optional service account to impersonate for gcloud commands. If this service account is not specified, the module will use Application Default Credentials. | `string` | `""` | no |
-| key\_file | The GCP Service Account credentials file path used to deploy ASM. | `string` | `""` | no |
-| location | The location (zone or region) this cluster has been created in. | `string` | n/a | yes |
-| managed\_control\_plane | ASM managed control plane boolean. Determines whether to install ASM managed control plane. Installing ASM managed control plane does not install gateways. Documentation on how to install gateways with ASM MCP can be found at https://cloud.google.com/service-mesh/docs/managed-control-plane#install_istio_gateways_optional. | `bool` | `false` | no |
-| mode | ASM mode for deployment. Supported modes are `install` and `upgrade`. | `string` | `"install"` | no |
-| options | Comma separated list of options. Works with in-cluster control plane only. Supported options are documented in https://cloud.google.com/service-mesh/docs/enable-optional-features. | `list(any)` | `[]` | no |
-| outdir | Sets `--outdir` option. | `string` | `"none"` | no |
+| enable\_cni | Determines whether to enable CNI for this ASM installation. | `bool` | `false` | no |
+| enable\_cross\_cluster\_service\_discovery | Determines whether to enable cross-cluster service discovery between this cluster and other clusters in the fleet. | `bool` | `false` | no |
+| enable\_mdp | Determines whether to enable Managed Data Plane (MDP) for this ASM installation. | `bool` | `false` | no |
+| mesh\_config | MeshConfig specifies configuration available to the control plane. If unset the module will not attempt to create the MeshConfig (i.e. if managing this configuration elsewhere). The full list of options can be found at https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig | `map(any)` | `{}` | no |
 | project\_id | The project in which the resource belongs. | `string` | n/a | yes |
-| revision\_name | Sets `--revision-name` option. | `string` | `"none"` | no |
-| service\_account | The GCP Service Account email address used to deploy ASM. | `string` | `""` | no |
-| service\_account\_key\_file | Path to service account key file to auth as for running `gcloud container clusters get-credentials`. | `string` | `""` | no |
-| skip\_validation | Sets `_CI_NO_VALIDATE` variable. Determines whether the script should perform validation checks for prerequisites such as IAM roles, Google APIs etc. | `bool` | `false` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| asm\_wait | An output to use when you want to depend on ASM finishing |
+| revision\_name | The name of the installed managed ASM revision. |
 
- <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/asm/main.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2018 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,95 +14,76 @@
  * limitations under the License.
  */
 
-data "google_project" "asm_project" {
-  project_id = var.project_id
-}
-
 locals {
-  options_string         = length(var.options) > 0 ? join(",", var.options) : "none"
-  custom_overlays_string = length(var.custom_overlays) > 0 ? join(",", var.custom_overlays) : "none"
-  asm_git_tag_string     = (var.asm_git_tag == "" ? "none" : var.asm_git_tag)
-  service_account_string = (var.service_account == "" ? "none" : var.service_account)
-  key_file_string        = (var.key_file == "" ? "none" : var.key_file)
-  ca_cert                = lookup(var.ca_certs, "ca_cert", "none")
-  ca_key                 = lookup(var.ca_certs, "ca_key", "none")
-  root_cert              = lookup(var.ca_certs, "root_cert", "none")
-  cert_chain             = lookup(var.ca_certs, "cert_chain", "none")
-  revision_name_string   = (var.revision_name == "" ? "none" : var.revision_name)
-  asm_minor_version      = tonumber(split(".", var.asm_version)[1])
-  # https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages/blob/1cf61b679cd369f42a0e735f8e201de1a6a6433b/scripts/asm-installer/install_asm#L1970
-  iam_roles = [
-    "roles/container.admin",
-    "roles/meshconfig.admin",
-    "roles/gkehub.admin",
-  ]
-  # https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages/blob/1cf61b679cd369f42a0e735f8e201de1a6a6433b/scripts/asm-installer/install_asm#L1958
-  mcp_iam_roles = [
-    "roles/serviceusage.serviceUsageConsumer",
-    "roles/container.admin",
-    "roles/monitoring.metricWriter",
-    "roles/logging.logWriter",
-    "roles/gkehub.viewer",
-    "roles/gkehub.gatewayAdmin",
-  ]
-  # if enable_gcp_iam_roles is set, grant IAM roles to first non null principal in the order below
-  asm_iam_member = var.enable_gcp_iam_roles ? coalesce(var.impersonate_service_account, var.service_account, var.iam_member) : ""
-  # compute any additonal resources that ASM provisioner should depend on
-  additional_depends_on = concat(var.enable_gcp_apis ? [module.asm-services[0].project_id] : [], local.asm_iam_member != "" ? [for k, v in google_project_iam_member.asm_iam : v.etag] : [])
-  # base command template for ASM installation
-  kubectl_create_command_base = "${path.module}/scripts/install_asm.sh ${var.project_id} ${var.cluster_name} ${var.location} ${var.asm_version} ${var.mode} ${var.managed_control_plane} ${var.skip_validation} ${local.options_string} ${local.custom_overlays_string} ${var.enable_all} ${var.enable_cluster_roles} ${var.enable_cluster_labels} ${var.enable_gcp_components} ${var.enable_registration} ${var.outdir} ${var.ca} ${local.ca_cert} ${local.ca_key} ${local.root_cert} ${local.cert_chain} ${local.service_account_string} ${local.key_file_string} ${local.asm_git_tag_string} ${local.revision_name_string}"
+  // GKE release channel is a list with max length 1 https://github.com/hashicorp/terraform-provider-google/blob/9d5f69f9f0f74f1a8245f1a52dd6cffb572bbce4/google/resource_container_cluster.go#L954
+  gke_release_channel       = data.google_container_cluster.asm_cluster.release_channel != null ? data.google_container_cluster.asm_cluster.release_channel[0].channel : ""
+  gke_release_channel_fixed = local.gke_release_channel == "UNSPECIFIED" ? "" : local.gke_release_channel
+  // In order or precedence, use (1) user specified channel, (2) GKE release channel, and (3) regular channel
+  channel          = lower(coalesce(var.channel, local.gke_release_channel_fixed, "regular"))
+  revision_name    = "asm-managed${local.channel == "regular" ? "" : "-${local.channel}"}"
+  mesh_config_name = "istio-${local.revision_name}"
+  // CNI should be enabled if either enable_cni or enable_mdp are set
+  enable_cni = var.enable_cni || var.enable_mdp
 }
 
-resource "google_project_iam_member" "asm_iam" {
-  for_each = toset(local.asm_iam_member != "" ? (var.managed_control_plane ? local.mcp_iam_roles : local.iam_roles) : [])
+data "google_container_cluster" "asm_cluster" {
   project  = var.project_id
-  role     = each.value
-  member   = "serviceAccount:${local.asm_iam_member}"
+  name     = var.cluster_name
+  location = var.cluster_location
 }
 
-module "asm-services" {
-  source  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version = "~> 11.3"
+resource "kubernetes_namespace" "system_namespace" {
+  metadata {
+    name = "istio-system"
+  }
+}
 
-  count = var.enable_gcp_apis ? 1 : 0
+resource "kubernetes_config_map" "mesh_config" {
+  count = length(var.mesh_config) == 0 ? 0 : 1
 
-  project_id                  = var.project_id
-  disable_services_on_destroy = false
-  disable_dependent_services  = false
+  metadata {
+    name      = local.mesh_config_name
+    namespace = kubernetes_namespace.system_namespace.metadata[0].name
+    labels = {
+      "istio.io/rev" = local.revision_name
+    }
+  }
+
+  data = {
+    mesh = yamlencode(var.mesh_config)
+  }
+}
 
-  # https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages/blob/1cf61b679cd369f42a0e735f8e201de1a6a6433b/scripts/asm-installer/install_asm#L2005
-  activate_apis = [
-    "container.googleapis.com",
-    "monitoring.googleapis.com",
-    "logging.googleapis.com",
-    "cloudtrace.googleapis.com",
-    "meshtelemetry.googleapis.com",
-    "meshconfig.googleapis.com",
-    "meshca.googleapis.com",
-    "iamcredentials.googleapis.com",
-    "gkeconnect.googleapis.com",
-    "gkehub.googleapis.com",
-    "cloudresourcemanager.googleapis.com",
-    "stackdriver.googleapis.com",
-  ]
+resource "kubernetes_config_map" "asm_options" {
+  metadata {
+    name      = "asm-options"
+    namespace = kubernetes_namespace.system_namespace.metadata[0].name
 }
 
-module "asm_install" {
-  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
-  version = "~> 3.1"
 
-  module_depends_on = concat([var.cluster_endpoint], local.additional_depends_on)
+  data = {
+    CROSS_CLUSTER_SERVICE_DISCOVERY = var.enable_cross_cluster_service_discovery ? "ON" : "OFF"
+  }
+}
+
+# TODO(Monkeyanator) due to a bug in the gke_hub_feature resource implementation this fails when enabling an enabled
+# feature (i.e. running the module twice) and does not disable the feature upon TF destroy.
+#resource "google_gke_hub_feature" "mesh_feature" {
+#  name     = "servicemesh"
+#  project  = var.project_id
+#  location = "global"
+#  provider = google-beta
+#}
+
+module "cpr" {
+  source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+
+  project_id       = var.project_id
+  cluster_name     = var.cluster_name
+  cluster_location = var.cluster_location
 
-  gcloud_sdk_version          = var.gcloud_sdk_version
-  upgrade                     = true
-  additional_components       = ["kubectl", "kpt", "beta"]
-  cluster_name                = var.cluster_name
-  cluster_location            = var.location
-  project_id                  = var.project_id
-  service_account_key_file    = var.service_account_key_file
-  impersonate_service_account = var.impersonate_service_account
+  kubectl_create_command  = "${path.module}/scripts/create_cpr.sh ${local.revision_name} ${local.channel} ${local.enable_cni}"
+  kubectl_destroy_command = "${path.module}/scripts/destroy_cpr.sh ${local.revision_name}"
 
-  # enable_namespace_creation flag is only available starting 1.10
-  kubectl_create_command  = (local.asm_minor_version > 9 ? "${local.kubectl_create_command_base} ${var.enable_namespace_creation}" : local.kubectl_create_command_base)
-  kubectl_destroy_command = "${path.module}/scripts/destroy_asm.sh"
+  module_depends_on = [kubernetes_config_map.mesh_config, kubernetes_config_map.asm_options]
 }

modules/asm/outputs.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2018 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-output "asm_wait" {
-  description = "An output to use when you want to depend on ASM finishing"
-  value       = module.asm_install.wait
+output "revision_name" {
+  value       = local.revision_name
+  description = "The name of the installed managed ASM revision."
 }

modules/asm/scripts/create_cpr.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+if [ "$#" -lt 3 ]; then
+    >&2 echo "Not all expected arguments set."
+    exit 1
+fi
+
+# Wait for the CRD to get created before creating the CPR.
+readonly CPR_RESOURCE=controlplanerevisions.mesh.cloud.google.com
+for _i in {1..18}; do
+  echo "Ensuring ControlPlaneRevision exists in cluster... attempt ${_i}"
+  if kubectl get crd "${CPR_RESOURCE}"
+  then
+    break
+  else
+    sleep 10
+  fi
+done
+
+kubectl wait --for condition=established --timeout=60s crd/"${CPR_RESOURCE}"
+
+REVISION_NAME=$1; shift
+CHANNEL=$1; shift
+ENABLE_CNI=$1; shift
+
+cat <<EOF | kubectl apply -f -
+apiVersion: mesh.cloud.google.com/v1beta1
+kind: ControlPlaneRevision
+metadata:
+  name: "${REVISION_NAME}"
+  namespace: istio-system
+  labels:
+    mesh.cloud.google.com/managed-cni-enabled: "${ENABLE_CNI}"
+spec:
+  type: managed_service
+  channel: "${CHANNEL}"
+EOF
+
+kubectl wait -n istio-system --for=condition=Reconciled controlplanerevision/"${REVISION_NAME}" --timeout 10m