feat!: Promote node sysctl config to GA (#1536)

Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: Andrew Peabody <[email protected]> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Bharath KKB <[email protected]>
terraform-google-modules · Feb 3, 2023 · 754f4e3 · 754f4e3
1 parent 24c09ec
commit 754f4e3
Show file tree

Hide file tree

Showing 33 changed files with 136 additions and 30 deletions.
diff --git a/README.md b/README.md
@@ -188,6 +188,7 @@ Then perform the following commands on the root folder:
 | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no |
 | node\_pools | List of maps containing node pools | `list(map(any))` | <pre>[<br>  {<br>    "name": "default-node-pool"<br>  }<br>]</pre> | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
+| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | <pre>{<br>  "all": [<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ],<br>  "default-node-pool": []<br>}</pre> | no |
 | node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
@@ -310,7 +311,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.13+
-- [Terraform Provider for GCP][terraform-provider-google] v4.47
+- [Terraform Provider for GCP][terraform-provider-google] v4.51
 #### gcloud
 Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH.
 See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information.

diff --git a/autogen/main/README.md b/autogen/main/README.md
@@ -256,9 +256,9 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.13+
 {% if beta_cluster %}
-- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47
+- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51
 {% else %}
-- [Terraform Provider for GCP][terraform-provider-google] v4.47
+- [Terraform Provider for GCP][terraform-provider-google] v4.51
 {% endif %}
 #### gcloud
 Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH.

diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -803,6 +803,7 @@ resource "google_container_node_pool" "windows_pools" {
         cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
       }
     }
+    {% endif %}
 
     {% if i == 0 %}
     dynamic "linux_node_config" {
@@ -819,7 +820,6 @@ resource "google_container_node_pool" "windows_pools" {
       }
     }
     {% endif %}
-    {% endif %}
 
     boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "")
 

diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
@@ -191,7 +191,6 @@ variable "node_pools_metadata" {
     default-node-pool = {}
   }
 }
-{% if beta_cluster %}
 
 variable "node_pools_linux_node_configs_sysctls" {
   type        = map(map(string))
@@ -204,7 +203,6 @@ variable "node_pools_linux_node_configs_sysctls" {
   }
 }
 {% endif %}
-{% endif %}
 
 variable "enable_cost_allocation" {
   type        = bool

diff --git a/autogen/main/variables_defaults.tf.tmpl b/autogen/main/variables_defaults.tf.tmpl
@@ -104,7 +104,6 @@ locals {
     ),
     var.node_pools_oauth_scopes
   )
-  {% if beta_cluster %}
 
   node_pools_linux_node_configs_sysctls = merge(
     { all = {} },
@@ -115,6 +114,5 @@ locals {
     ),
     var.node_pools_linux_node_configs_sysctls
   )
-  {% endif %}
 }
 {% endif %}
diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl
@@ -24,7 +24,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.47.0, < 5.0"
+      version = ">= 4.51.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"
@@ -38,7 +38,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.47.0, < 5.0"
+      version = ">= 4.51.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

diff --git a/cluster.tf b/cluster.tf
@@ -499,6 +499,20 @@ resource "google_container_node_pool" "pools" {
     }
 
 
+    dynamic "linux_node_config" {
+      for_each = length(merge(
+        local.node_pools_linux_node_configs_sysctls["all"],
+        local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+      )) != 0 ? [1] : []
+
+      content {
+        sysctls = merge(
+          local.node_pools_linux_node_configs_sysctls["all"],
+          local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+        )
+      }
+    }
+
     boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "")
 
     shielded_instance_config {
@@ -657,6 +671,7 @@ resource "google_container_node_pool" "windows_pools" {
     }
 
 
+
     boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "")
 
     shielded_instance_config {

diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md
@@ -187,7 +187,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.13+
-- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47
+- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51
 #### gcloud
 Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH.
 See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information.

diff --git a/modules/beta-autopilot-private-cluster/versions.tf b/modules/beta-autopilot-private-cluster/versions.tf
@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.47.0, < 5.0"
+      version = ">= 4.51.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md
@@ -174,7 +174,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.13+
-- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47
+- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51
 #### gcloud
 Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH.
 See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information.

diff --git a/modules/beta-autopilot-public-cluster/versions.tf b/modules/beta-autopilot-public-cluster/versions.tf
@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.47.0, < 5.0"
+      version = ">= 4.51.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
@@ -379,7 +379,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.13+
-- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47
+- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51
 #### gcloud
 Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH.
 See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information.

diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf
@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.47.0, < 5.0"
+      version = ">= 4.51.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
@@ -357,7 +357,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.13+
-- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47
+- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51
 #### gcloud
 Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH.
 See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information.

diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf
@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.47.0, < 5.0"
+      version = ">= 4.51.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md
@@ -366,7 +366,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.13+
-- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47
+- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51
 #### gcloud
 Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH.
 See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information.

diff --git a/modules/beta-public-cluster-update-variant/versions.tf b/modules/beta-public-cluster-update-variant/versions.tf
@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.47.0, < 5.0"
+      version = ">= 4.51.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
@@ -344,7 +344,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.13+
-- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47
+- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51
 #### gcloud
 Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH.
 See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information.

diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf
@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.47.0, < 5.0"
+      version = ">= 4.51.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
@@ -221,6 +221,7 @@ Then perform the following commands on the root folder:
 | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no |
 | node\_pools | List of maps containing node pools | `list(map(any))` | <pre>[<br>  {<br>    "name": "default-node-pool"<br>  }<br>]</pre> | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
+| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | <pre>{<br>  "all": [<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ],<br>  "default-node-pool": []<br>}</pre> | no |
 | node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
@@ -345,7 +346,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.13+
-- [Terraform Provider for GCP][terraform-provider-google] v4.47
+- [Terraform Provider for GCP][terraform-provider-google] v4.51
 #### gcloud
 Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH.
 See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information.

diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf
@@ -612,6 +612,20 @@ resource "google_container_node_pool" "pools" {
     }
 
 
+    dynamic "linux_node_config" {
+      for_each = length(merge(
+        local.node_pools_linux_node_configs_sysctls["all"],
+        local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+      )) != 0 ? [1] : []
+
+      content {
+        sysctls = merge(
+          local.node_pools_linux_node_configs_sysctls["all"],
+          local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+        )
+      }
+    }
+
     boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "")
 
     shielded_instance_config {
@@ -771,6 +785,7 @@ resource "google_container_node_pool" "windows_pools" {
     }
 
 
+
     boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "")
 
     shielded_instance_config {

diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf
@@ -191,6 +191,17 @@ variable "node_pools_metadata" {
   }
 }
 
+variable "node_pools_linux_node_configs_sysctls" {
+  type        = map(map(string))
+  description = "Map of maps containing linux node config sysctls by node-pool name"
+
+  # Default is being set in variables_defaults.tf
+  default = {
+    all               = {}
+    default-node-pool = {}
+  }
+}
+
 variable "enable_cost_allocation" {
   type        = bool
   description = "Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery"

diff --git a/modules/private-cluster-update-variant/variables_defaults.tf b/modules/private-cluster-update-variant/variables_defaults.tf
@@ -103,4 +103,14 @@ locals {
     ),
     var.node_pools_oauth_scopes
   )
+
+  node_pools_linux_node_configs_sysctls = merge(
+    { all = {} },
+    { default-node-pool = {} },
+    zipmap(
+      [for node_pool in var.node_pools : node_pool["name"]],
+      [for node_pool in var.node_pools : {}]
+    ),
+    var.node_pools_linux_node_configs_sysctls
+  )
 }
diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf
@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.47.0, < 5.0"
+      version = ">= 4.51.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
@@ -199,6 +199,7 @@ Then perform the following commands on the root folder:
 | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no |
 | node\_pools | List of maps containing node pools | `list(map(any))` | <pre>[<br>  {<br>    "name": "default-node-pool"<br>  }<br>]</pre> | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
+| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | <pre>{<br>  "all": [<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ],<br>  "default-node-pool": []<br>}</pre> | no |
 | node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
@@ -323,7 +324,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.13+
-- [Terraform Provider for GCP][terraform-provider-google] v4.47
+- [Terraform Provider for GCP][terraform-provider-google] v4.51
 #### gcloud
 Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH.
 See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information.

diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf
@@ -518,6 +518,20 @@ resource "google_container_node_pool" "pools" {
     }
 
 
+    dynamic "linux_node_config" {
+      for_each = length(merge(
+        local.node_pools_linux_node_configs_sysctls["all"],
+        local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+      )) != 0 ? [1] : []
+
+      content {
+        sysctls = merge(
+          local.node_pools_linux_node_configs_sysctls["all"],
+          local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+        )
+      }
+    }
+
     boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "")
 
     shielded_instance_config {
@@ -676,6 +690,7 @@ resource "google_container_node_pool" "windows_pools" {
     }
 
 
+
     boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "")
 
     shielded_instance_config {

diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf
@@ -191,6 +191,17 @@ variable "node_pools_metadata" {
   }
 }
 
+variable "node_pools_linux_node_configs_sysctls" {
+  type        = map(map(string))
+  description = "Map of maps containing linux node config sysctls by node-pool name"
+
+  # Default is being set in variables_defaults.tf
+  default = {
+    all               = {}
+    default-node-pool = {}
+  }
+}
+
 variable "enable_cost_allocation" {
   type        = bool
   description = "Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery"

diff --git a/modules/private-cluster/variables_defaults.tf b/modules/private-cluster/variables_defaults.tf
@@ -103,4 +103,14 @@ locals {
     ),
     var.node_pools_oauth_scopes
   )
+
+  node_pools_linux_node_configs_sysctls = merge(
+    { all = {} },
+    { default-node-pool = {} },
+    zipmap(
+      [for node_pool in var.node_pools : node_pool["name"]],
+      [for node_pool in var.node_pools : {}]
+    ),
+    var.node_pools_linux_node_configs_sysctls
+  )
 }