diff --git a/CHANGELOG.md b/CHANGELOG.md index 31cb1f8d0f..e9467d2a27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,11 @@ Extending the adopted spec, each change should have a link to its corresponding ## [Unreleased] -### Added +## [2.0.0] 2019-06-ZZ + +### Changed +* Supported version of Terraform is 0.12. [#58] * Add configuration flag for enable BinAuthZ Admission controller [#160] * Add configuration flag for `pod_security_policy_config` [#163] * Support for a guest accelerator in node pool configuration. [#157] diff --git a/Gemfile b/Gemfile index 2fffe26f1f..a54d14ec29 100644 --- a/Gemfile +++ b/Gemfile @@ -15,7 +15,7 @@ ruby "~> 2.5" source 'https://rubygems.org/' do - gem "kitchen-terraform", "~> 4.0" + gem "kitchen-terraform", "~> 4.9" gem "kubeclient", "~> 4.0" gem "rest-client", "~> 2.0" end diff --git a/Makefile b/Makefile index 6c481c10c2..adab2ec226 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ SHELL := /usr/bin/env bash # Docker build config variables CREDENTIALS_PATH ?= /cft/workdir/credentials.json DOCKER_ORG := gcr.io/cloud-foundation-cicd -DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 1.3.0 +DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 2.0.0 DOCKER_REPO_BASE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform:${DOCKER_TAG_BASE_KITCHEN_TERRAFORM} DOCKER_TAG_KITCHEN_TERRAFORM ?= ${DOCKER_TAG_BASE_KITCHEN_TERRAFORM} DOCKER_IMAGE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform_terraform-google-kubernetes-engine diff --git a/README.md b/README.md index e56d7db123..4f48a96b50 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ module "gke" { all = {} default-node-pool = { - default-node-pool = "true" + default-node-pool = true } } @@ -74,7 +74,7 @@ module "gke" { default-node-pool = [ { key = "default-node-pool" - value = "true" + value = true effect = "PREFER_NO_SCHEDULE" }, ] @@ -109,75 +109,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. [^]: (autogen_docs_start) - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | string | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | string | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created. | string | `"create"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - [^]: (autogen_docs_end) ## Requirements @@ -195,8 +126,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.11.x -- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.3, v2.6, v2.7 +- [Terraform](https://www.terraform.io/downloads.html) 0.12.x +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.8 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/auth.tf b/auth.tf index 5ad4160145..48e7cc6a5f 100644 --- a/auth.tf +++ b/auth.tf @@ -20,7 +20,7 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - provider = "google" + provider = google } /****************************************** @@ -29,6 +29,6 @@ data "google_client_config" "default" { provider "kubernetes" { load_config_file = false host = "https://${local.cluster_endpoint}" - token = "${data.google_client_config.default.access_token}" - cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(local.cluster_ca_certificate) } diff --git a/autogen/README.md b/autogen/README.md index 778bfdb38e..1e3b4a44fa 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -66,7 +66,7 @@ module "gke" { all = {} default-node-pool = { - default-node-pool = "true" + default-node-pool = true } } @@ -84,7 +84,7 @@ module "gke" { default-node-pool = [ { key = "default-node-pool" - value = "true" + value = true effect = "PREFER_NO_SCHEDULE" }, ] @@ -136,11 +136,11 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.11.x +- [Terraform](https://www.terraform.io/downloads.html) 0.12.x {% if private_cluster %} -- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.3, v2.6, v2.7 +- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.8 {% else %} -- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.3, v2.6, v2.7 +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.8 {% endif %} ### Configure a Service Account diff --git a/autogen/auth.tf b/autogen/auth.tf index 3e961cd6b1..d480409119 100644 --- a/autogen/auth.tf +++ b/autogen/auth.tf @@ -20,7 +20,8 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = {% if private_cluster %}google-beta{%else %}google{% endif %} + } /****************************************** @@ -29,6 +30,6 @@ data "google_client_config" "default" { provider "kubernetes" { load_config_file = false host = "https://${local.cluster_endpoint}" - token = "${data.google_client_config.default.access_token}" - cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(local.cluster_ca_certificate) } diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index e055f639dd..130ca7b312 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -20,74 +20,99 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" - count = "${var.regional ? 1 : 0}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" - - region = "${var.region}" - node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] - - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + {% if private_cluster %} + provider = google-beta + {% else %} + provider = google + {% endif %} + count = var.regional ? 1 : 0 + name = var.name + description = var.description + project = var.project_id + + region = var.region + node_locations = coalescelist( + compact(var.zones), + sort(random_shuffle.available_zones.result), + ) + + network = data.google_compute_network.gke_network.self_link network_policy { - enabled = "${var.network_policy}" - provider = "${var.network_policy_provider}" + enabled = var.network_policy + provider = var.network_policy_provider } - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_regional}" + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_regional - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + logging_service = var.logging_service + monitoring_service = var.monitoring_service {% if private_cluster %} - enable_binary_authorization = "${var.enable_binary_authorization}" - pod_security_policy_config = "${var.pod_security_policy_config}" + enable_binary_authorization = var.enable_binary_authorization + + dynamic "pod_security_policy_config" { + for_each = var.pod_security_policy_config + content { + enabled = pod_security_policy_config.value.enabled + } + } + {% endif %} - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -98,73 +123,127 @@ resource "google_container_cluster" "primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) } } -{% if private_cluster %} +{% if private_cluster %} private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block } -{% endif %} - remove_default_node_pool = "${var.remove_default_node_pool}" +{% endif %} + remove_default_node_pool = var.remove_default_node_pool } /****************************************** Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { - provider = "google-beta" - count = "${var.regional ? length(var.node_pools) : 0}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - region = "${var.region}" - cluster = "${google_container_cluster.primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? length(var.node_pools) : 0 + name = var.node_pools[count.index]["name"] + project = var.project_id + region = var.region + cluster = google_container_cluster.primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_regional, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" - - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] - - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -175,16 +254,19 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_regional_cluster" { - count = "${var.regional ? 1 : 0}" + count = var.regional ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] + depends_on = [ + google_container_cluster.primary, + google_container_node_pool.pools, + ] } diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index a1ae54b19f..590406023c 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -20,74 +20,96 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" - count = "${var.regional ? 0 : 1}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + {% if private_cluster %} + provider = google-beta + {% else %} + provider = google + {% endif %} + count = var.regional ? 0 : 1 + name = var.name + description = var.description + project = var.project_id - zone = "${var.zones[0]}" - node_locations = ["${slice(var.zones,1,length(var.zones))}"] + zone = var.zones[0] + node_locations = slice(var.zones, 1, length(var.zones)) - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + network = data.google_compute_network.gke_network.self_link network_policy { - enabled = "${var.network_policy}" - provider = "${var.network_policy_provider}" + enabled = var.network_policy + provider = var.network_policy_provider } - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_zonal}" - - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_zonal + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } {% if private_cluster %} - enable_binary_authorization = "${var.enable_binary_authorization}" - pod_security_policy_config = "${var.pod_security_policy_config}" + enable_binary_authorization = var.enable_binary_authorization + + dynamic "pod_security_policy_config" { + for_each = var.pod_security_policy_config + content { + enabled = pod_security_policy_config.value.enabled + } + } {% endif %} - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -98,73 +120,128 @@ resource "google_container_cluster" "zonal_primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) } } -{% if private_cluster %} +{% if private_cluster %} private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block } {% endif %} - remove_default_node_pool = "${var.remove_default_node_pool}" + remove_default_node_pool = var.remove_default_node_pool } /****************************************** Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - provider = "google-beta" - count = "${var.regional ? 0 : length(var.node_pools)}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - zone = "${var.zones[0]}" - cluster = "${google_container_cluster.zonal_primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? 0 : length(var.node_pools) + name = var.node_pools[count.index]["name"] + project = var.project_id + zone = var.zones[0] + cluster = google_container_cluster.zonal_primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_zonal, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" - - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] - - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -175,16 +252,19 @@ resource "google_container_node_pool" "zonal_pools" { } resource "null_resource" "wait_for_zonal_cluster" { - count = "${var.regional ? 0 : 1}" + count = var.regional ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } diff --git a/autogen/dns.tf b/autogen/dns.tf index 1b0d83eb23..cdfbde7589 100644 --- a/autogen/dns.tf +++ b/autogen/dns.tf @@ -20,35 +20,48 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = "${local.custom_kube_dns_config ? 1 : 0}" + count = local.custom_kube_dns_config ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" } - depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + data.google_client_config.default, + google_container_cluster.primary, + google_container_node_pool.pools, + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } /****************************************** Create kube-dns confimap *****************************************/ resource "kubernetes_config_map" "kube-dns" { - count = "${local.custom_kube_dns_config ? 1 : 0}" + count = local.custom_kube_dns_config ? 1 : 0 metadata { name = "kube-dns" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { stubDomains = < 0 + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id - cluster_type = "${var.regional ? "regional" : "zonal"}" + cluster_type = var.regional ? "regional" : "zonal" cluster_type_output_name = { - regional = "${element(concat(google_container_cluster.primary.*.name, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.name, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.name, [""]), + 0, + ) } cluster_type_output_location = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.zone, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.zone, [""]), + 0, + ) } cluster_type_output_region = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${var.region}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = var.region } - cluster_type_output_regional_zones = "${flatten(google_container_cluster.primary.*.node_locations)}" - cluster_type_output_zonal_zones = "${slice(var.zones, 1, length(var.zones))}" + cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) + cluster_type_output_zonal_zones = slice(var.zones, 1, length(var.zones)) cluster_type_output_zones = { - regional = "${local.cluster_type_output_regional_zones}" - zonal = "${concat(google_container_cluster.zonal_primary.*.zone, local.cluster_type_output_zonal_zones)}" + regional = local.cluster_type_output_regional_zones + zonal = concat( + google_container_cluster.zonal_primary.*.zone, + local.cluster_type_output_zonal_zones, + ) } {% if private_cluster %} cluster_type_output_endpoint = { - regional = "${ - var.deploy_using_private_endpoint ? - element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : - element(concat(google_container_cluster.primary.*.endpoint, list("")), 0) - }" - - zonal = "${ - var.deploy_using_private_endpoint ? - element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : - element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0) - }" + regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : element(concat(google_container_cluster.primary.*.endpoint, list("")), 0) + + zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0) } {% else %} cluster_type_output_endpoint = { - regional = "${element(concat(google_container_cluster.primary.*.endpoint, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.endpoint, [""]), + 0, + ) } {% endif %} cluster_type_output_master_auth = { - regional = "${concat(google_container_cluster.primary.*.master_auth, list())}" - zonal = "${concat(google_container_cluster.zonal_primary.*.master_auth, list())}" + regional = concat(google_container_cluster.primary.*.master_auth, []) + zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) } cluster_type_output_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.master_version, + [""], + ), + 0, + ) } cluster_type_output_min_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.min_master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.min_master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.min_master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.min_master_version, + [""], + ), + 0, + ) } cluster_type_output_logging_service = { - regional = "${element(concat(google_container_cluster.primary.*.logging_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.logging_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.logging_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.logging_service, + [""], + ), + 0, + ) } cluster_type_output_monitoring_service = { - regional = "${element(concat(google_container_cluster.primary.*.monitoring_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.monitoring_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.monitoring_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.monitoring_service, + [""], + ), + 0, + ) } cluster_type_output_network_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_http_load_balancing_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_kubernetes_dashboard_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_node_pools_names = { - regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" + regional = concat(google_container_node_pool.pools.*.name, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) } cluster_type_output_node_pools_versions = { - regional = "${concat(google_container_node_pool.pools.*.version, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.version, list(""))}" + regional = concat(google_container_node_pool.pools.*.version, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) } {% if private_cluster %} cluster_type_output_pod_security_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, list("")), 0) + zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, list("")), 0) } {% endif %} - cluster_master_auth_list_layer1 = "${local.cluster_type_output_master_auth[local.cluster_type]}" - cluster_master_auth_list_layer2 = "${local.cluster_master_auth_list_layer1[0]}" - cluster_master_auth_map = "${local.cluster_master_auth_list_layer2[0]}" + cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] + cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = "${local.cluster_type_output_name[local.cluster_type]}" - cluster_location = "${local.cluster_type_output_location[local.cluster_type]}" - cluster_region = "${local.cluster_type_output_region[local.cluster_type]}" - cluster_zones = "${sort(local.cluster_type_output_zones[local.cluster_type])}" - cluster_endpoint = "${local.cluster_type_output_endpoint[local.cluster_type]}" - cluster_ca_certificate = "${lookup(local.cluster_master_auth_map, "cluster_ca_certificate")}" - cluster_master_version = "${local.cluster_type_output_master_version[local.cluster_type]}" - cluster_min_master_version = "${local.cluster_type_output_min_master_version[local.cluster_type]}" - cluster_logging_service = "${local.cluster_type_output_logging_service[local.cluster_type]}" - cluster_monitoring_service = "${local.cluster_type_output_monitoring_service[local.cluster_type]}" - cluster_node_pools_names = "${local.cluster_type_output_node_pools_names[local.cluster_type]}" - cluster_node_pools_versions = "${local.cluster_type_output_node_pools_versions[local.cluster_type]}" - - cluster_network_policy_enabled = "${local.cluster_type_output_network_policy_enabled[local.cluster_type] ? false : true}" - cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] ? false : true}" - cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] ? false : true}" - cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] ? false : true}" + cluster_name = local.cluster_type_output_name[local.cluster_type] + cluster_location = local.cluster_type_output_location[local.cluster_type] + cluster_region = local.cluster_type_output_region[local.cluster_type] + cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) + cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] + cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] + cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] + cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] + cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] + cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] + cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] + + cluster_network_policy_enabled = !local.cluster_type_output_network_policy_enabled[local.cluster_type] + cluster_http_load_balancing_enabled = !local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] + cluster_horizontal_pod_autoscaling_enabled = !local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] + cluster_kubernetes_dashboard_enabled = !local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] {% if private_cluster %} - cluster_pod_security_policy_enabled = "${local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] ? true : false}" + cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] {% endif %} } @@ -177,9 +265,9 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = "google-beta" - region = "${var.region}" - project = "${var.project_id}" + provider = google-beta + region = var.region + project = var.project_id } data "google_container_engine_versions" "zone" { @@ -187,7 +275,7 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + zone = var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0] - project = "${var.project_id}" + project = var.project_id } diff --git a/autogen/masq.tf b/autogen/masq.tf index e8856a8bf0..374f541ac8 100644 --- a/autogen/masq.tf +++ b/autogen/masq.tf @@ -20,18 +20,18 @@ Create ip-masq-agent confimap *****************************************/ resource "kubernetes_config_map" "ip-masq-agent" { - count = "${var.network_policy ? 1 : 0}" + count = var.network_policy ? 1 : 0 metadata { name = "ip-masq-agent" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { config = < 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -163,16 +231,19 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_regional_cluster" { - count = "${var.regional ? 1 : 0}" + count = var.regional ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] + depends_on = [ + google_container_cluster.primary, + google_container_node_pool.pools, + ] } diff --git a/cluster_zonal.tf b/cluster_zonal.tf index 9b214db43b..4295eb82ac 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -20,70 +20,82 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "google" - count = "${var.regional ? 0 : 1}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + provider = google + count = var.regional ? 0 : 1 + name = var.name + description = var.description + project = var.project_id - zone = "${var.zones[0]}" - node_locations = ["${slice(var.zones,1,length(var.zones))}"] + zone = var.zones[0] + node_locations = slice(var.zones, 1, length(var.zones)) - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + network = data.google_compute_network.gke_network.self_link network_policy { - enabled = "${var.network_policy}" - provider = "${var.network_policy_provider}" + enabled = var.network_policy + provider = var.network_policy_provider } - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_zonal}" - - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_zonal + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -94,65 +106,121 @@ resource "google_container_cluster" "zonal_primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) } } - remove_default_node_pool = "${var.remove_default_node_pool}" + + remove_default_node_pool = var.remove_default_node_pool } /****************************************** Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - provider = "google-beta" - count = "${var.regional ? 0 : length(var.node_pools)}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - zone = "${var.zones[0]}" - cluster = "${google_container_cluster.zonal_primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? 0 : length(var.node_pools) + name = var.node_pools[count.index]["name"] + project = var.project_id + zone = var.zones[0] + cluster = google_container_cluster.zonal_primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_zonal, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" - - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] - - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -163,16 +231,19 @@ resource "google_container_node_pool" "zonal_pools" { } resource "null_resource" "wait_for_zonal_cluster" { - count = "${var.regional ? 0 : 1}" + count = var.regional ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } diff --git a/dns.tf b/dns.tf index 25effe580a..4d37fcef0c 100644 --- a/dns.tf +++ b/dns.tf @@ -20,35 +20,48 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = "${local.custom_kube_dns_config ? 1 : 0}" + count = local.custom_kube_dns_config ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" } - depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + data.google_client_config.default, + google_container_cluster.primary, + google_container_node_pool.pools, + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } /****************************************** Create kube-dns confimap *****************************************/ resource "kubernetes_config_map" "kube-dns" { - count = "${local.custom_kube_dns_config ? 1 : 0}" + count = local.custom_kube_dns_config ? 1 : 0 metadata { name = "kube-dns" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { stubDomains = < 0 + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id - cluster_type = "${var.regional ? "regional" : "zonal"}" + cluster_type = var.regional ? "regional" : "zonal" cluster_type_output_name = { - regional = "${element(concat(google_container_cluster.primary.*.name, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.name, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.name, [""]), + 0, + ) } cluster_type_output_location = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.zone, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.zone, [""]), + 0, + ) } cluster_type_output_region = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${var.region}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = var.region } - cluster_type_output_regional_zones = "${flatten(google_container_cluster.primary.*.node_locations)}" - cluster_type_output_zonal_zones = "${slice(var.zones, 1, length(var.zones))}" + cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) + cluster_type_output_zonal_zones = slice(var.zones, 1, length(var.zones)) cluster_type_output_zones = { - regional = "${local.cluster_type_output_regional_zones}" - zonal = "${concat(google_container_cluster.zonal_primary.*.zone, local.cluster_type_output_zonal_zones)}" + regional = local.cluster_type_output_regional_zones + zonal = concat( + google_container_cluster.zonal_primary.*.zone, + local.cluster_type_output_zonal_zones, + ) } cluster_type_output_endpoint = { - regional = "${element(concat(google_container_cluster.primary.*.endpoint, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.endpoint, [""]), + 0, + ) } cluster_type_output_master_auth = { - regional = "${concat(google_container_cluster.primary.*.master_auth, list())}" - zonal = "${concat(google_container_cluster.zonal_primary.*.master_auth, list())}" + regional = concat(google_container_cluster.primary.*.master_auth, []) + zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) } cluster_type_output_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.master_version, + [""], + ), + 0, + ) } cluster_type_output_min_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.min_master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.min_master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.min_master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.min_master_version, + [""], + ), + 0, + ) } cluster_type_output_logging_service = { - regional = "${element(concat(google_container_cluster.primary.*.logging_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.logging_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.logging_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.logging_service, + [""], + ), + 0, + ) } cluster_type_output_monitoring_service = { - regional = "${element(concat(google_container_cluster.primary.*.monitoring_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.monitoring_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.monitoring_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.monitoring_service, + [""], + ), + 0, + ) } cluster_type_output_network_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_http_load_balancing_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_kubernetes_dashboard_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_node_pools_names = { - regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" + regional = concat(google_container_node_pool.pools.*.name, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) } cluster_type_output_node_pools_versions = { - regional = "${concat(google_container_node_pool.pools.*.version, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.version, list(""))}" + regional = concat(google_container_node_pool.pools.*.version, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) } - cluster_master_auth_list_layer1 = "${local.cluster_type_output_master_auth[local.cluster_type]}" - cluster_master_auth_list_layer2 = "${local.cluster_master_auth_list_layer1[0]}" - cluster_master_auth_map = "${local.cluster_master_auth_list_layer2[0]}" + cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] + cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = "${local.cluster_type_output_name[local.cluster_type]}" - cluster_location = "${local.cluster_type_output_location[local.cluster_type]}" - cluster_region = "${local.cluster_type_output_region[local.cluster_type]}" - cluster_zones = "${sort(local.cluster_type_output_zones[local.cluster_type])}" - cluster_endpoint = "${local.cluster_type_output_endpoint[local.cluster_type]}" - cluster_ca_certificate = "${lookup(local.cluster_master_auth_map, "cluster_ca_certificate")}" - cluster_master_version = "${local.cluster_type_output_master_version[local.cluster_type]}" - cluster_min_master_version = "${local.cluster_type_output_min_master_version[local.cluster_type]}" - cluster_logging_service = "${local.cluster_type_output_logging_service[local.cluster_type]}" - cluster_monitoring_service = "${local.cluster_type_output_monitoring_service[local.cluster_type]}" - cluster_node_pools_names = "${local.cluster_type_output_node_pools_names[local.cluster_type]}" - cluster_node_pools_versions = "${local.cluster_type_output_node_pools_versions[local.cluster_type]}" - - cluster_network_policy_enabled = "${local.cluster_type_output_network_policy_enabled[local.cluster_type] ? false : true}" - cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] ? false : true}" - cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] ? false : true}" - cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] ? false : true}" + cluster_name = local.cluster_type_output_name[local.cluster_type] + cluster_location = local.cluster_type_output_location[local.cluster_type] + cluster_region = local.cluster_type_output_region[local.cluster_type] + cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) + cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] + cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] + cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] + cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] + cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] + cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] + cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] + + cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] + cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] + cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] } /****************************************** Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = "google-beta" - region = "${var.region}" - project = "${var.project_id}" + provider = google-beta + region = var.region + project = var.project_id } data "google_container_engine_versions" "zone" { @@ -161,7 +257,7 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + zone = var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0] - project = "${var.project_id}" + project = var.project_id } diff --git a/masq.tf b/masq.tf index a78d263cef..7475b904de 100644 --- a/masq.tf +++ b/masq.tf @@ -20,18 +20,18 @@ Create ip-masq-agent confimap *****************************************/ resource "kubernetes_config_map" "ip-masq-agent" { - count = "${var.network_policy ? 1 : 0}" + count = var.network_policy ? 1 : 0 metadata { name = "ip-masq-agent" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { config = <
### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | string | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created. | string | `"create"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - [^]: (autogen_docs_end) ## Requirements @@ -207,8 +131,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.11.x -- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.3, v2.6, v2.7 +- [Terraform](https://www.terraform.io/downloads.html) 0.12.x +- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.8 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/modules/private-cluster/auth.tf b/modules/private-cluster/auth.tf index 0bbafaf4a2..c177eee5a7 100644 --- a/modules/private-cluster/auth.tf +++ b/modules/private-cluster/auth.tf @@ -20,7 +20,7 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - provider = "google-beta" + provider = google-beta } /****************************************** @@ -29,6 +29,6 @@ data "google_client_config" "default" { provider "kubernetes" { load_config_file = false host = "https://${local.cluster_endpoint}" - token = "${data.google_client_config.default.access_token}" - cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(local.cluster_ca_certificate) } diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf index dc22a7b912..4552e24166 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster_regional.tf @@ -20,72 +20,93 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "google-beta" - count = "${var.regional ? 1 : 0}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + provider = google-beta + count = var.regional ? 1 : 0 + name = var.name + description = var.description + project = var.project_id - region = "${var.region}" - node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] + region = var.region + node_locations = coalescelist( + compact(var.zones), + sort(random_shuffle.available_zones.result), + ) - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + network = data.google_compute_network.gke_network.self_link network_policy { - enabled = "${var.network_policy}" - provider = "${var.network_policy_provider}" + enabled = var.network_policy + provider = var.network_policy_provider } - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_regional}" + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_regional - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + logging_service = var.logging_service + monitoring_service = var.monitoring_service - enable_binary_authorization = "${var.enable_binary_authorization}" - pod_security_policy_config = "${var.pod_security_policy_config}" - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + enable_binary_authorization = var.enable_binary_authorization + + dynamic "pod_security_policy_config" { + for_each = var.pod_security_policy_config + content { + enabled = pod_security_policy_config.value.enabled + } + } + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -96,71 +117,125 @@ resource "google_container_cluster" "primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) } } private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block } - remove_default_node_pool = "${var.remove_default_node_pool}" + remove_default_node_pool = var.remove_default_node_pool } /****************************************** Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { - provider = "google-beta" - count = "${var.regional ? length(var.node_pools) : 0}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - region = "${var.region}" - cluster = "${google_container_cluster.primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? length(var.node_pools) : 0 + name = var.node_pools[count.index]["name"] + project = var.project_id + region = var.region + cluster = google_container_cluster.primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_regional, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" - - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] - - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -171,16 +246,19 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_regional_cluster" { - count = "${var.regional ? 1 : 0}" + count = var.regional ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] + depends_on = [ + google_container_cluster.primary, + google_container_node_pool.pools, + ] } diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf index b451df9f7c..7a3fe7f76b 100644 --- a/modules/private-cluster/cluster_zonal.tf +++ b/modules/private-cluster/cluster_zonal.tf @@ -20,72 +20,90 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "google-beta" - count = "${var.regional ? 0 : 1}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + provider = google-beta + count = var.regional ? 0 : 1 + name = var.name + description = var.description + project = var.project_id - zone = "${var.zones[0]}" - node_locations = ["${slice(var.zones,1,length(var.zones))}"] + zone = var.zones[0] + node_locations = slice(var.zones, 1, length(var.zones)) - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + network = data.google_compute_network.gke_network.self_link network_policy { - enabled = "${var.network_policy}" - provider = "${var.network_policy_provider}" + enabled = var.network_policy + provider = var.network_policy_provider } - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_zonal}" + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_zonal + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + enable_binary_authorization = var.enable_binary_authorization - enable_binary_authorization = "${var.enable_binary_authorization}" - pod_security_policy_config = "${var.pod_security_policy_config}" - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + dynamic "pod_security_policy_config" { + for_each = var.pod_security_policy_config + content { + enabled = pod_security_policy_config.value.enabled + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -96,71 +114,126 @@ resource "google_container_cluster" "zonal_primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) } } private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block } - remove_default_node_pool = "${var.remove_default_node_pool}" + remove_default_node_pool = var.remove_default_node_pool } /****************************************** Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - provider = "google-beta" - count = "${var.regional ? 0 : length(var.node_pools)}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - zone = "${var.zones[0]}" - cluster = "${google_container_cluster.zonal_primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? 0 : length(var.node_pools) + name = var.node_pools[count.index]["name"] + project = var.project_id + zone = var.zones[0] + cluster = google_container_cluster.zonal_primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_zonal, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" - - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] - - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -171,16 +244,19 @@ resource "google_container_node_pool" "zonal_pools" { } resource "null_resource" "wait_for_zonal_cluster" { - count = "${var.regional ? 0 : 1}" + count = var.regional ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf index 25effe580a..4d37fcef0c 100644 --- a/modules/private-cluster/dns.tf +++ b/modules/private-cluster/dns.tf @@ -20,35 +20,48 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = "${local.custom_kube_dns_config ? 1 : 0}" + count = local.custom_kube_dns_config ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" } - depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + data.google_client_config.default, + google_container_cluster.primary, + google_container_node_pool.pools, + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } /****************************************** Create kube-dns confimap *****************************************/ resource "kubernetes_config_map" "kube-dns" { - count = "${local.custom_kube_dns_config ? 1 : 0}" + count = local.custom_kube_dns_config ? 1 : 0 metadata { name = "kube-dns" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { stubDomains = < 0 + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id - cluster_type = "${var.regional ? "regional" : "zonal"}" + cluster_type = var.regional ? "regional" : "zonal" cluster_type_output_name = { - regional = "${element(concat(google_container_cluster.primary.*.name, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.name, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.name, [""]), + 0, + ) } cluster_type_output_location = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.zone, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.zone, [""]), + 0, + ) } cluster_type_output_region = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${var.region}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = var.region } - cluster_type_output_regional_zones = "${flatten(google_container_cluster.primary.*.node_locations)}" - cluster_type_output_zonal_zones = "${slice(var.zones, 1, length(var.zones))}" + cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) + cluster_type_output_zonal_zones = slice(var.zones, 1, length(var.zones)) cluster_type_output_zones = { - regional = "${local.cluster_type_output_regional_zones}" - zonal = "${concat(google_container_cluster.zonal_primary.*.zone, local.cluster_type_output_zonal_zones)}" + regional = local.cluster_type_output_regional_zones + zonal = concat( + google_container_cluster.zonal_primary.*.zone, + local.cluster_type_output_zonal_zones, + ) } cluster_type_output_endpoint = { - regional = "${ - var.deploy_using_private_endpoint ? - element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : - element(concat(google_container_cluster.primary.*.endpoint, list("")), 0) - }" + regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : element(concat(google_container_cluster.primary.*.endpoint, list("")), 0) - zonal = "${ - var.deploy_using_private_endpoint ? - element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : - element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0) - }" + zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0) } cluster_type_output_master_auth = { - regional = "${concat(google_container_cluster.primary.*.master_auth, list())}" - zonal = "${concat(google_container_cluster.zonal_primary.*.master_auth, list())}" + regional = concat(google_container_cluster.primary.*.master_auth, []) + zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) } cluster_type_output_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.master_version, + [""], + ), + 0, + ) } cluster_type_output_min_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.min_master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.min_master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.min_master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.min_master_version, + [""], + ), + 0, + ) } cluster_type_output_logging_service = { - regional = "${element(concat(google_container_cluster.primary.*.logging_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.logging_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.logging_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.logging_service, + [""], + ), + 0, + ) } cluster_type_output_monitoring_service = { - regional = "${element(concat(google_container_cluster.primary.*.monitoring_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.monitoring_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.monitoring_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.monitoring_service, + [""], + ), + 0, + ) } cluster_type_output_network_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_http_load_balancing_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_kubernetes_dashboard_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_node_pools_names = { - regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" + regional = concat(google_container_node_pool.pools.*.name, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) } cluster_type_output_node_pools_versions = { - regional = "${concat(google_container_node_pool.pools.*.version, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.version, list(""))}" + regional = concat(google_container_node_pool.pools.*.version, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) } cluster_type_output_pod_security_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, list("")), 0) + zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, list("")), 0) } - cluster_master_auth_list_layer1 = "${local.cluster_type_output_master_auth[local.cluster_type]}" - cluster_master_auth_list_layer2 = "${local.cluster_master_auth_list_layer1[0]}" - cluster_master_auth_map = "${local.cluster_master_auth_list_layer2[0]}" + cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] + cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = "${local.cluster_type_output_name[local.cluster_type]}" - cluster_location = "${local.cluster_type_output_location[local.cluster_type]}" - cluster_region = "${local.cluster_type_output_region[local.cluster_type]}" - cluster_zones = "${sort(local.cluster_type_output_zones[local.cluster_type])}" - cluster_endpoint = "${local.cluster_type_output_endpoint[local.cluster_type]}" - cluster_ca_certificate = "${lookup(local.cluster_master_auth_map, "cluster_ca_certificate")}" - cluster_master_version = "${local.cluster_type_output_master_version[local.cluster_type]}" - cluster_min_master_version = "${local.cluster_type_output_min_master_version[local.cluster_type]}" - cluster_logging_service = "${local.cluster_type_output_logging_service[local.cluster_type]}" - cluster_monitoring_service = "${local.cluster_type_output_monitoring_service[local.cluster_type]}" - cluster_node_pools_names = "${local.cluster_type_output_node_pools_names[local.cluster_type]}" - cluster_node_pools_versions = "${local.cluster_type_output_node_pools_versions[local.cluster_type]}" - - cluster_network_policy_enabled = "${local.cluster_type_output_network_policy_enabled[local.cluster_type] ? false : true}" - cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] ? false : true}" - cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] ? false : true}" - cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] ? false : true}" - cluster_pod_security_policy_enabled = "${local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] ? false : true}" + cluster_name = local.cluster_type_output_name[local.cluster_type] + cluster_location = local.cluster_type_output_location[local.cluster_type] + cluster_region = local.cluster_type_output_region[local.cluster_type] + cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) + cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] + cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] + cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] + cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] + cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] + cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] + cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] + + cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] + cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] + cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] } /****************************************** Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = "google-beta" - region = "${var.region}" - project = "${var.project_id}" + provider = google-beta + region = var.region + project = var.project_id } data "google_container_engine_versions" "zone" { @@ -176,7 +261,7 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + zone = var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0] - project = "${var.project_id}" + project = var.project_id } diff --git a/modules/private-cluster/masq.tf b/modules/private-cluster/masq.tf index a78d263cef..7475b904de 100644 --- a/modules/private-cluster/masq.tf +++ b/modules/private-cluster/masq.tf @@ -20,18 +20,18 @@ Create ip-masq-agent confimap *****************************************/ resource "kubernetes_config_map" "ip-masq-agent" { - count = "${var.network_policy ? 1 : 0}" + count = var.network_policy ? 1 : 0 metadata { name = "ip-masq-agent" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { config = < /dev/null && terraform validate "{}" && terraform fmt -check=true -write=false "{}"' } # This function runs 'go fmt' and 'go vet' on every file diff --git a/variables.tf b/variables.tf index 2723a39df5..087227c0d1 100644 --- a/variables.tf +++ b/variables.tf @@ -17,58 +17,68 @@ // This file was automatically generated from a template in ./autogen variable "project_id" { + type = string description = "The project ID to host the cluster in (required)" } variable "name" { + type = string description = "The name of the cluster (required)" } variable "description" { + type = string description = "The description of the cluster" default = "" } variable "regional" { + type = bool description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" default = true } variable "region" { + type = string description = "The region to host the cluster in (required)" } variable "zones" { - type = "list" + type = list(string) description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" - default = [""] + default = [] } variable "network" { + type = string description = "The VPC network to host the cluster in (required)" } variable "network_project_id" { + type = string description = "The project ID of the shared VPC's host (for shared vpc support)" default = "" } variable "subnetwork" { + type = string description = "The subnetwork to host the cluster in (required)" } variable "kubernetes_version" { + type = string description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region." default = "latest" } variable "node_version" { + type = string description = "The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation." default = "" } variable "master_authorized_networks_config" { - type = "list" + type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) description = <