feat(TPG>=5.44.2)!: add standard cluster support for `insecureKubelet…

…ReadonlyPortEnabled` (#2082)
terraform-google-modules · Oct 16, 2024 · 96626d5 · 96626d5
1 parent 66231d5
commit 96626d5
Show file tree

Hide file tree

Showing 34 changed files with 217 additions and 111 deletions.
diff --git a/README.md b/README.md
@@ -190,6 +190,7 @@ Then perform the following commands on the root folder:
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |
@@ -319,6 +320,7 @@ The node_pools variable takes the following parameters:
 | gpu_partition_size | Size of partitions to create on the GPU | null | Optional |
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
+| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional |
 | key | The key required for the taint | | Required |
 | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |

diff --git a/autogen/main/README.md b/autogen/main/README.md
@@ -213,6 +213,7 @@ The node_pools variable takes the following parameters:
 | gpu_partition_size | Size of partitions to create on the GPU | null | Optional |
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
+| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional |
 | key | The key required for the taint | | Required |
 | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |

diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -661,7 +661,6 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
-  {% if beta_cluster %}
 
   node_pool_defaults {
     node_config_defaults {
@@ -675,15 +674,17 @@ resource "google_container_cluster" "primary" {
       }
       {% endif %}
       {% if autopilot_cluster != true %}
+      {% if beta_cluster %}
       gcfs_config {
         enabled = var.enable_gcfs
       }
       {% endif %}
+      insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null
+      {% endif %}
     }
   }
-  {% endif %}
-  {% if beta_cluster %}
 
+  {% if beta_cluster %}
   depends_on = [google_project_iam_member.service_agent]
   {% endif %}
 }
@@ -1046,14 +1047,15 @@ resource "google_container_node_pool" "windows_pools" {
     dynamic "kubelet_config" {
       for_each = length(setintersection(
         keys(each.value),
-        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
       )) != 0 ? [1] : []
 
       content {
-        cpu_manager_policy   = lookup(each.value, "cpu_manager_policy", "static")
-        cpu_cfs_quota        = lookup(each.value, "cpu_cfs_quota", null)
-        cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
-        pod_pids_limit       = lookup(each.value, "pod_pids_limit", null)
+        cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
+        cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
+        cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null)
+        pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }
     {% if beta_cluster %}

diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
@@ -109,6 +109,12 @@ variable "service_external_ips" {
 }
 
 {% if autopilot_cluster != true %}
+variable "insecure_kubelet_readonly_port_enabled" {
+  type = bool
+  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
+  default     = null
+}
+
 variable "datapath_provider" {
   type        = string
   description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature."

diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl
@@ -24,11 +24,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 5.40.0, < 7"
+      version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 5.40.0, < 7"
+      version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"
@@ -89,7 +89,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 5.40.0, < 7"
+      version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

diff --git a/cluster.tf b/cluster.tf
@@ -500,6 +500,13 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
+
+  node_pool_defaults {
+    node_config_defaults {
+      insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null
+    }
+  }
+
 }
 /******************************************
   Create Container Cluster node pools
@@ -739,14 +746,15 @@ resource "google_container_node_pool" "pools" {
     dynamic "kubelet_config" {
       for_each = length(setintersection(
         keys(each.value),
-        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
       )) != 0 ? [1] : []
 
       content {
-        cpu_manager_policy   = lookup(each.value, "cpu_manager_policy", "static")
-        cpu_cfs_quota        = lookup(each.value, "cpu_cfs_quota", null)
-        cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
-        pod_pids_limit       = lookup(each.value, "pod_pids_limit", null)
+        cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
+        cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
+        cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null)
+        pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }
 
@@ -1029,14 +1037,15 @@ resource "google_container_node_pool" "windows_pools" {
     dynamic "kubelet_config" {
       for_each = length(setintersection(
         keys(each.value),
-        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
       )) != 0 ? [1] : []
 
       content {
-        cpu_manager_policy   = lookup(each.value, "cpu_manager_policy", "static")
-        cpu_cfs_quota        = lookup(each.value, "cpu_cfs_quota", null)
-        cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
-        pod_pids_limit       = lookup(each.value, "pod_pids_limit", null)
+        cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
+        cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
+        cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null)
+        pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }
 

diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
@@ -67,20 +67,21 @@ module "gke" {
       service_account   = var.compute_engine_service_account
     },
     {
-      name                      = "pool-03"
-      machine_type              = "n1-standard-2"
-      node_locations            = "${var.region}-b,${var.region}-c"
-      autoscaling               = false
-      node_count                = 2
-      disk_type                 = "pd-standard"
-      auto_upgrade              = true
-      service_account           = var.compute_engine_service_account
-      pod_range                 = "test"
-      sandbox_enabled           = true
-      cpu_manager_policy        = "static"
-      cpu_cfs_quota             = true
-      local_ssd_ephemeral_count = 2
-      pod_pids_limit            = 4096
+      name                                   = "pool-03"
+      machine_type                           = "n1-standard-2"
+      node_locations                         = "${var.region}-b,${var.region}-c"
+      autoscaling                            = false
+      node_count                             = 2
+      disk_type                              = "pd-standard"
+      auto_upgrade                           = true
+      service_account                        = var.compute_engine_service_account
+      pod_range                              = "test"
+      sandbox_enabled                        = true
+      cpu_manager_policy                     = "static"
+      cpu_cfs_quota                          = true
+      insecure_kubelet_readonly_port_enabled = "FALSE"
+      local_ssd_ephemeral_count              = 2
+      pod_pids_limit                         = 4096
     },
     {
       name                = "pool-04"

diff --git a/examples/node_pool_update_variant/main.tf b/examples/node_pool_update_variant/main.tf
@@ -61,11 +61,12 @@ module "gke" {
 
   node_pools = [
     {
-      name            = "pool-01"
-      min_count       = 1
-      max_count       = 2
-      service_account = var.compute_engine_service_account
-      auto_upgrade    = true
+      name                                   = "pool-01"
+      min_count                              = 1
+      max_count                              = 2
+      service_account                        = var.compute_engine_service_account
+      auto_upgrade                           = true
+      insecure_kubelet_readonly_port_enabled = "FALSE"
     },
     {
       name              = "pool-02"

diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
@@ -233,6 +233,7 @@ Then perform the following commands on the root folder:
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |
@@ -377,6 +378,7 @@ The node_pools variable takes the following parameters:
 | gpu_partition_size | Size of partitions to create on the GPU | null | Optional |
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
+| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional |
 | key | The key required for the taint | | Required |
 | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |

diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -581,6 +581,7 @@ resource "google_container_cluster" "primary" {
       gcfs_config {
         enabled = var.enable_gcfs
       }
+      insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null
     }
   }
 
@@ -915,14 +916,15 @@ resource "google_container_node_pool" "pools" {
     dynamic "kubelet_config" {
       for_each = length(setintersection(
         keys(each.value),
-        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
       )) != 0 ? [1] : []
 
       content {
-        cpu_manager_policy   = lookup(each.value, "cpu_manager_policy", "static")
-        cpu_cfs_quota        = lookup(each.value, "cpu_cfs_quota", null)
-        cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
-        pod_pids_limit       = lookup(each.value, "pod_pids_limit", null)
+        cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
+        cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
+        cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null)
+        pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }
 
@@ -1219,14 +1221,15 @@ resource "google_container_node_pool" "windows_pools" {
     dynamic "kubelet_config" {
       for_each = length(setintersection(
         keys(each.value),
-        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"]
+        ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"]
       )) != 0 ? [1] : []
 
       content {
-        cpu_manager_policy   = lookup(each.value, "cpu_manager_policy", "static")
-        cpu_cfs_quota        = lookup(each.value, "cpu_cfs_quota", null)
-        cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
-        pod_pids_limit       = lookup(each.value, "pod_pids_limit", null)
+        cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
+        cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
+        cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != null ? var.insecure_kubelet_readonly_port_enabled : null)
+        pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }
 

diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
@@ -108,6 +108,12 @@ variable "service_external_ips" {
   default     = false
 }
 
+variable "insecure_kubelet_readonly_port_enabled" {
+  type        = bool
+  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
+  default     = null
+}
+
 variable "datapath_provider" {
   type        = string
   description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature."

diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf
@@ -21,11 +21,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 5.40.0, < 7"
+      version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 5.40.0, < 7"
+      version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"