From 904e925f977a5a4142e47a4e190780b4b37f858d Mon Sep 17 00:00:00 2001 From: Jonathan Meyers Date: Wed, 2 Mar 2022 09:10:07 -1000 Subject: [PATCH] feat: GKE autopilot support (#1148) * adding auto-pilot support * fixes * add kitchen tests * Update main.tf * fix: add back in * Update examples/simple_autopilot_private/README.md Co-authored-by: Bharath KKB * Update examples/simple_autopilot_private/README.md Co-authored-by: Bharath KKB * Update modules/beta-autopilot-public-cluster/versions.tf Co-authored-by: Bharath KKB * Update examples/simple_autopilot_private/main.tf Co-authored-by: Bharath KKB * update dates and remove Vars * fixes * i hate symlinks * add vars and outputs * docs generation * add random string to subnet names * Update main.tf * adding auto-pilot support * fixes * add kitchen tests * Update main.tf * fix: add back in * Update examples/simple_autopilot_private/README.md Co-authored-by: Bharath KKB * Update examples/simple_autopilot_private/README.md Co-authored-by: Bharath KKB * Update modules/beta-autopilot-public-cluster/versions.tf Co-authored-by: Bharath KKB * Update examples/simple_autopilot_private/main.tf Co-authored-by: Bharath KKB * update dates and remove Vars * fixes * i hate symlinks * add vars and outputs * docs generation * add random string to subnet names * Update main.tf * remove random name Co-authored-by: Corey McGalliard Co-authored-by: Bharath KKB --- .kitchen.yml | 14 + README.md | 4 +- autogen/main/README.md | 21 +- autogen/main/cluster.tf.tmpl | 59 +- autogen/main/dns.tf.tmpl | 10 +- autogen/main/firewall.tf.tmpl | 2 +- autogen/main/main.tf.tmpl | 20 +- autogen/main/masq.tf.tmpl | 4 +- autogen/main/networks.tf | 2 +- autogen/main/outputs.tf.tmpl | 10 +- autogen/main/sa.tf.tmpl | 2 +- autogen/main/variables.tf.tmpl | 16 +- autogen/main/variables_defaults.tf | 4 +- autogen/main/versions.tf.tmpl | 2 +- autogen/safer-cluster/main.tf.tmpl | 2 +- autogen/safer-cluster/outputs.tf.tmpl | 2 +- autogen/safer-cluster/variables.tf.tmpl | 2 +- autogen/safer-cluster/versions.tf.tmpl | 2 +- autogen_modules.json | 20 + build/int.cloudbuild.yaml | 30 + cluster.tf | 13 +- dns.tf | 2 +- .../terraform/terraform.tfvars | 8 +- .../terraform/terraform.tfvars | 8 +- .../terraform/terraform.tfvars | 8 +- examples/simple_autopilot_private/README.md | 35 ++ examples/simple_autopilot_private/main.tf | 58 ++ examples/simple_autopilot_private/network.tf | 49 ++ examples/simple_autopilot_private/outputs.tf | 70 +++ .../simple_autopilot_private/variables.tf | 24 + examples/simple_autopilot_private/versions.tf | 28 + examples/simple_autopilot_public/README.md | 35 ++ examples/simple_autopilot_public/main.tf | 47 ++ examples/simple_autopilot_public/network.tf | 49 ++ examples/simple_autopilot_public/outputs.tf | 70 +++ examples/simple_autopilot_public/variables.tf | 24 + examples/simple_autopilot_public/versions.tf | 28 + firewall.tf | 2 +- main.tf | 4 +- masq.tf | 2 +- .../beta-autopilot-private-cluster/README.md | 235 ++++++++ .../beta-autopilot-private-cluster/auth.tf | 24 + .../beta-autopilot-private-cluster/cluster.tf | 176 ++++++ modules/beta-autopilot-private-cluster/dns.tf | 119 ++++ .../firewall.tf | 198 +++++++ .../beta-autopilot-private-cluster/main.tf | 183 ++++++ .../beta-autopilot-private-cluster/masq.tf | 46 ++ .../networks.tf | 26 + .../beta-autopilot-private-cluster/outputs.tf | 174 ++++++ modules/beta-autopilot-private-cluster/sa.tf | 86 +++ .../scripts/delete-default-resource.sh | 41 ++ .../scripts/kubectl_wrapper.sh | 53 ++ .../variables.tf | 554 ++++++++++++++++++ .../variables_defaults.tf | 21 + .../versions.tf | 34 ++ .../beta-autopilot-public-cluster/README.md | 222 +++++++ modules/beta-autopilot-public-cluster/auth.tf | 24 + .../beta-autopilot-public-cluster/cluster.tf | 157 +++++ modules/beta-autopilot-public-cluster/dns.tf | 119 ++++ .../beta-autopilot-public-cluster/firewall.tf | 207 +++++++ modules/beta-autopilot-public-cluster/main.tf | 182 ++++++ modules/beta-autopilot-public-cluster/masq.tf | 46 ++ .../beta-autopilot-public-cluster/networks.tf | 26 + .../beta-autopilot-public-cluster/outputs.tf | 164 ++++++ modules/beta-autopilot-public-cluster/sa.tf | 86 +++ .../scripts/delete-default-resource.sh | 41 ++ .../scripts/kubectl_wrapper.sh | 53 ++ .../variables.tf | 523 +++++++++++++++++ .../variables_defaults.tf | 21 + .../beta-autopilot-public-cluster/versions.tf | 34 ++ .../README.md | 10 +- .../cluster.tf | 14 +- .../dns.tf | 2 +- .../firewall.tf | 2 +- .../main.tf | 4 +- .../masq.tf | 2 +- .../networks.tf | 2 +- .../outputs.tf | 2 +- .../beta-private-cluster-update-variant/sa.tf | 2 +- .../variables.tf | 2 +- .../variables_defaults.tf | 2 +- .../versions.tf | 2 +- modules/beta-private-cluster/README.md | 10 +- modules/beta-private-cluster/cluster.tf | 14 +- modules/beta-private-cluster/dns.tf | 2 +- modules/beta-private-cluster/firewall.tf | 2 +- modules/beta-private-cluster/main.tf | 4 +- modules/beta-private-cluster/masq.tf | 2 +- modules/beta-private-cluster/networks.tf | 2 +- modules/beta-private-cluster/outputs.tf | 2 +- modules/beta-private-cluster/sa.tf | 2 +- modules/beta-private-cluster/variables.tf | 2 +- .../variables_defaults.tf | 2 +- modules/beta-private-cluster/versions.tf | 2 +- .../README.md | 10 +- .../cluster.tf | 14 +- .../beta-public-cluster-update-variant/dns.tf | 2 +- .../firewall.tf | 2 +- .../main.tf | 4 +- .../masq.tf | 2 +- .../networks.tf | 2 +- .../outputs.tf | 2 +- .../beta-public-cluster-update-variant/sa.tf | 2 +- .../variables.tf | 2 +- .../variables_defaults.tf | 2 +- .../versions.tf | 2 +- modules/beta-public-cluster/README.md | 10 +- modules/beta-public-cluster/cluster.tf | 14 +- modules/beta-public-cluster/dns.tf | 2 +- modules/beta-public-cluster/firewall.tf | 2 +- modules/beta-public-cluster/main.tf | 4 +- modules/beta-public-cluster/masq.tf | 2 +- modules/beta-public-cluster/networks.tf | 2 +- modules/beta-public-cluster/outputs.tf | 2 +- modules/beta-public-cluster/sa.tf | 2 +- modules/beta-public-cluster/variables.tf | 2 +- .../beta-public-cluster/variables_defaults.tf | 2 +- modules/beta-public-cluster/versions.tf | 2 +- .../private-cluster-update-variant/README.md | 4 +- .../private-cluster-update-variant/cluster.tf | 13 +- modules/private-cluster-update-variant/dns.tf | 2 +- .../firewall.tf | 2 +- .../private-cluster-update-variant/main.tf | 4 +- .../private-cluster-update-variant/masq.tf | 2 +- .../networks.tf | 2 +- .../private-cluster-update-variant/outputs.tf | 2 +- modules/private-cluster-update-variant/sa.tf | 2 +- .../variables.tf | 2 +- .../variables_defaults.tf | 2 +- .../versions.tf | 2 +- modules/private-cluster/README.md | 4 +- modules/private-cluster/cluster.tf | 13 +- modules/private-cluster/dns.tf | 2 +- modules/private-cluster/firewall.tf | 2 +- modules/private-cluster/main.tf | 4 +- modules/private-cluster/masq.tf | 2 +- modules/private-cluster/networks.tf | 2 +- modules/private-cluster/outputs.tf | 2 +- modules/private-cluster/sa.tf | 2 +- modules/private-cluster/variables.tf | 2 +- modules/private-cluster/variables_defaults.tf | 2 +- modules/private-cluster/versions.tf | 2 +- modules/safer-cluster-update-variant/main.tf | 2 +- .../safer-cluster-update-variant/outputs.tf | 2 +- .../safer-cluster-update-variant/variables.tf | 2 +- .../safer-cluster-update-variant/versions.tf | 2 +- modules/safer-cluster/main.tf | 2 +- modules/safer-cluster/outputs.tf | 2 +- modules/safer-cluster/variables.tf | 2 +- modules/safer-cluster/versions.tf | 2 +- networks.tf | 2 +- outputs.tf | 2 +- sa.tf | 2 +- .../simple_autopilot_private/example.tf | 22 + .../simple_autopilot_private/outputs.tf | 52 ++ .../simple_autopilot_private/variables.tf | 25 + .../simple_autopilot_public/example.tf | 22 + .../simple_autopilot_public/outputs.tf | 52 ++ .../simple_autopilot_public/variables.tf | 25 + .../controls/gcloud.rb | 68 +++ .../simple_autopilot_private/inspec.yml | 31 + .../controls/gcloud.rb | 64 ++ .../simple_autopilot_public/inspec.yml | 31 + test/setup/main.tf | 6 +- variables.tf | 2 +- variables_defaults.tf | 2 +- versions.tf | 2 +- 167 files changed, 5083 insertions(+), 272 deletions(-) create mode 100644 examples/simple_autopilot_private/README.md create mode 100644 examples/simple_autopilot_private/main.tf create mode 100644 examples/simple_autopilot_private/network.tf create mode 100644 examples/simple_autopilot_private/outputs.tf create mode 100644 examples/simple_autopilot_private/variables.tf create mode 100644 examples/simple_autopilot_private/versions.tf create mode 100644 examples/simple_autopilot_public/README.md create mode 100644 examples/simple_autopilot_public/main.tf create mode 100644 examples/simple_autopilot_public/network.tf create mode 100644 examples/simple_autopilot_public/outputs.tf create mode 100644 examples/simple_autopilot_public/variables.tf create mode 100644 examples/simple_autopilot_public/versions.tf create mode 100644 modules/beta-autopilot-private-cluster/README.md create mode 100644 modules/beta-autopilot-private-cluster/auth.tf create mode 100644 modules/beta-autopilot-private-cluster/cluster.tf create mode 100644 modules/beta-autopilot-private-cluster/dns.tf create mode 100644 modules/beta-autopilot-private-cluster/firewall.tf create mode 100644 modules/beta-autopilot-private-cluster/main.tf create mode 100644 modules/beta-autopilot-private-cluster/masq.tf create mode 100644 modules/beta-autopilot-private-cluster/networks.tf create mode 100644 modules/beta-autopilot-private-cluster/outputs.tf create mode 100644 modules/beta-autopilot-private-cluster/sa.tf create mode 100755 modules/beta-autopilot-private-cluster/scripts/delete-default-resource.sh create mode 100755 modules/beta-autopilot-private-cluster/scripts/kubectl_wrapper.sh create mode 100644 modules/beta-autopilot-private-cluster/variables.tf create mode 100644 modules/beta-autopilot-private-cluster/variables_defaults.tf create mode 100644 modules/beta-autopilot-private-cluster/versions.tf create mode 100644 modules/beta-autopilot-public-cluster/README.md create mode 100644 modules/beta-autopilot-public-cluster/auth.tf create mode 100644 modules/beta-autopilot-public-cluster/cluster.tf create mode 100644 modules/beta-autopilot-public-cluster/dns.tf create mode 100644 modules/beta-autopilot-public-cluster/firewall.tf create mode 100644 modules/beta-autopilot-public-cluster/main.tf create mode 100644 modules/beta-autopilot-public-cluster/masq.tf create mode 100644 modules/beta-autopilot-public-cluster/networks.tf create mode 100644 modules/beta-autopilot-public-cluster/outputs.tf create mode 100644 modules/beta-autopilot-public-cluster/sa.tf create mode 100755 modules/beta-autopilot-public-cluster/scripts/delete-default-resource.sh create mode 100755 modules/beta-autopilot-public-cluster/scripts/kubectl_wrapper.sh create mode 100644 modules/beta-autopilot-public-cluster/variables.tf create mode 100644 modules/beta-autopilot-public-cluster/variables_defaults.tf create mode 100644 modules/beta-autopilot-public-cluster/versions.tf create mode 100644 test/fixtures/simple_autopilot_private/example.tf create mode 100644 test/fixtures/simple_autopilot_private/outputs.tf create mode 100644 test/fixtures/simple_autopilot_private/variables.tf create mode 100644 test/fixtures/simple_autopilot_public/example.tf create mode 100644 test/fixtures/simple_autopilot_public/outputs.tf create mode 100644 test/fixtures/simple_autopilot_public/variables.tf create mode 100644 test/integration/simple_autopilot_private/controls/gcloud.rb create mode 100644 test/integration/simple_autopilot_private/inspec.yml create mode 100644 test/integration/simple_autopilot_public/controls/gcloud.rb create mode 100644 test/integration/simple_autopilot_public/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 6027399a4c..552f6eea59 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -244,3 +244,17 @@ suites: controls: - gcloud - kubectl + - name: "simple_autopilot_private" + driver: + root_module_directory: test/fixtures/simple_autopilot_private + verifier: + systems: + - name: simple_autopilot_private + backend: local + - name: "simple_autopilot_public" + driver: + root_module_directory: test/fixtures/simple_autopilot_public + verifier: + systems: + - name: simple_autopilot_public + backend: local diff --git a/README.md b/README.md index d4462fd12e..601c953a09 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,8 @@ module "gke" { ip_range_pods = "us-central1-01-gke-01-pods" ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false - horizontal_pod_autoscaling = true network_policy = false + horizontal_pod_autoscaling = true node_pools = [ { @@ -253,8 +253,6 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | - - ## Requirements Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: diff --git a/autogen/main/README.md b/autogen/main/README.md index f933b18d21..483ba08af8 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -72,20 +72,26 @@ module "gke" { subnetwork = "us-central1-01" ip_range_pods = "us-central1-01-gke-01-pods" ip_range_services = "us-central1-01-gke-01-services" + {% if autopilot_cluster != true %} http_load_balancing = false - horizontal_pod_autoscaling = true network_policy = false + {% endif %} + horizontal_pod_autoscaling = true {% if private_cluster %} enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" {% endif %} - {% if beta_cluster %} - istio = true - cloudrun = true - dns_cache = false + {% if beta_cluster and autopilot_cluster != true %} + istio = true + cloudrun = true + dns_cache = false + {% endif %} + {% if autopilot_cluster %} + enable_autopilot = true {% endif %} +{% if autopilot_cluster != true %} node_pools = [ { name = "default-node-pool" @@ -152,6 +158,7 @@ module "gke" { "default-node-pool", ] } +{% endif %} } ``` @@ -166,6 +173,7 @@ Then perform the following commands on the root folder: +{% if autopilot_cluster != true %} ## node_pools variable The node_pools variable takes the following parameters: @@ -220,8 +228,7 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | - - +{% endif %} ## Requirements Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 39a58e43da..741356f66b 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ resource "google_container_cluster" "primary" { node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = "projects/${local.network_project_id}/global/networks/${var.network}" - + {% if autopilot_cluster != true %} dynamic "network_policy" { for_each = local.cluster_network_policy @@ -44,6 +44,7 @@ resource "google_container_cluster" "primary" { provider = network_policy.value.provider } } + {% endif %} dynamic "release_channel" { for_each = local.release_channel @@ -64,13 +65,13 @@ resource "google_container_cluster" "primary" { subnetwork = "projects/${local.network_project_id}/regions/${local.region}/subnetworks/${var.subnetwork}" {% if beta_cluster %} - default_snat_status{ + default_snat_status { disabled = var.disable_default_snat } {% endif %} min_master_version = var.release_channel != null ? null : local.master_version -{% if beta_cluster %} +{% if beta_cluster and autopilot_cluster != true %} dynamic "cluster_telemetry" { for_each = local.cluster_telemetry_type_is_set ? [1] : [] content { @@ -98,7 +99,7 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service {% endif %} - + {% if autopilot_cluster != true %} cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -107,14 +108,14 @@ resource "google_container_cluster" "primary" { content { service_account = local.service_account oauth_scopes = local.node_pools_oauth_scopes["all"] -{% if beta_cluster %} + {% if beta_cluster %} min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") -{% endif %} + {% endif %} } } -{% if beta_cluster %} + {% if beta_cluster %} autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" -{% endif %} + {% endif %} dynamic "resource_limits" { for_each = local.autoscaling_resource_limits content { @@ -124,16 +125,15 @@ resource "google_container_cluster" "primary" { } } } - + {% endif %} vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling } - + {% if autopilot_cluster != true %} default_max_pods_per_node = var.default_max_pods_per_node - enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization -{% if beta_cluster %} + {% if beta_cluster %} enable_intranode_visibility = var.enable_intranode_visibility enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu @@ -153,6 +153,10 @@ resource "google_container_cluster" "primary" { } enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting + {% endif %} + {% endif %} +{% if autopilot_cluster %} + enable_autopilot = true {% endif %} dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config @@ -181,14 +185,15 @@ resource "google_container_cluster" "primary" { horizontal_pod_autoscaling { disabled = !var.horizontal_pod_autoscaling } - + {% if autopilot_cluster != true %} network_policy_config { disabled = !var.network_policy } - {% if beta_cluster %} + {% endif %} + {% if beta_cluster and autopilot_cluster != true %} istio_config { - disabled = ! var.istio + disabled = !var.istio auth = var.istio_auth } @@ -258,7 +263,6 @@ resource "google_container_cluster" "primary" { end_time = maintenance_exclusion.value.end_time } } - {% else %} daily_maintenance_window { start_time = var.maintenance_start_time @@ -266,16 +270,18 @@ resource "google_container_cluster" "primary" { {% endif %} } + {% if autopilot_cluster != true %} lifecycle { ignore_changes = [node_pool, initial_node_count, resource_labels["asmv"], resource_labels["mesh_id"]] } + {% endif %} timeouts { create = "45m" update = "45m" delete = "45m" } - + {% if autopilot_cluster != true %} node_pool { name = "default-pool" initial_node_count = var.initial_node_count @@ -321,6 +327,7 @@ resource "google_container_cluster" "primary" { } } } + {% endif %} dynamic "resource_usage_export_config" { for_each = var.resource_usage_export_dataset_id != "" ? [{ @@ -362,6 +369,7 @@ resource "google_container_cluster" "primary" { } {% endif %} + {% if autopilot_cluster != true %} remove_default_node_pool = var.remove_default_node_pool dynamic "database_encryption" { @@ -380,27 +388,30 @@ resource "google_container_cluster" "primary" { workload_pool = workload_identity_config.value.workload_pool } } + {% endif %} + {% if autopilot_cluster != true %} dynamic "authenticator_groups_config" { for_each = local.cluster_authenticator_security_group content { security_group = authenticator_groups_config.value.security_group } } - -{% if beta_cluster %} + {% endif %} + {% if beta_cluster %} notification_config { pubsub { enabled = var.notification_config_topic != "" ? true : false - topic = var.notification_config_topic + topic = var.notification_config_topic } } -{% endif %} + {% endif %} } - +{% if autopilot_cluster != true %} /****************************************** Create Container Cluster node pools *****************************************/ +{% endif %} {% if update_variant %} locals { force_node_pool_recreation_resources = [ @@ -491,6 +502,7 @@ resource "random_id" "name" { } {% endif %} +{% if autopilot_cluster != true %} resource "google_container_node_pool" "pools" { {% if beta_cluster %} provider = google-beta @@ -698,3 +710,4 @@ resource "google_container_node_pool" "pools" { delete = "45m" } } +{% endif %} diff --git a/autogen/main/dns.tf.tmpl b/autogen/main/dns.tf.tmpl index edba110f4f..00297d6876 100644 --- a/autogen/main/dns.tf.tmpl +++ b/autogen/main/dns.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,9 @@ module "gcloud_delete_default_kube_dns_configmap" { module_depends_on = concat( [google_container_cluster.primary.master_version], + {% if autopilot_cluster != true %} [for pool in google_container_node_pool.pools : pool.name] + {% endif %} ) } @@ -63,7 +65,9 @@ EOF depends_on = [ module.gcloud_delete_default_kube_dns_configmap.wait, google_container_cluster.primary, + {% if autopilot_cluster != true %} google_container_node_pool.pools, + {% endif %} ] } @@ -89,7 +93,9 @@ EOF depends_on = [ module.gcloud_delete_default_kube_dns_configmap.wait, google_container_cluster.primary, + {% if autopilot_cluster != true %} google_container_node_pool.pools, + {% endif %} ] } @@ -118,6 +124,8 @@ EOF depends_on = [ module.gcloud_delete_default_kube_dns_configmap.wait, google_container_cluster.primary, + {% if autopilot_cluster != true %} google_container_node_pool.pools, + {% endif %} ] } diff --git a/autogen/main/firewall.tf.tmpl b/autogen/main/firewall.tf.tmpl index 183e761f70..aa0f413166 100644 --- a/autogen/main/firewall.tf.tmpl +++ b/autogen/main/firewall.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index 0b587c9524..ab24310091 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,10 +48,11 @@ locals { master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - + {% if autopilot_cluster != true %} // Build a map of maps of node pools from a list of objects node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) + {% endif %} release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] @@ -82,6 +83,7 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} +{% if autopilot_cluster != true %} cluster_network_policy = var.network_policy ? [{ enabled = true provider = var.network_policy_provider @@ -89,7 +91,7 @@ locals { enabled = false provider = null }] - +{% endif %} {% if beta_cluster %} cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type @@ -156,8 +158,10 @@ locals { cidr_blocks : var.master_authorized_networks }] + {% if autopilot_cluster != true %} cluster_output_node_pools_names = concat([for np in google_container_node_pool.pools : np.name], [""]) cluster_output_node_pools_versions = { for np in google_container_node_pool.pools : np.name => np.version } + {% endif %} cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] @@ -174,12 +178,14 @@ locals { cluster_min_master_version = local.cluster_output_min_master_version cluster_logging_service = local.cluster_output_logging_service cluster_monitoring_service = local.cluster_output_monitoring_service + {% if autopilot_cluster != true %} cluster_node_pools_names = local.cluster_output_node_pools_names cluster_node_pools_versions = local.cluster_output_node_pools_versions - cluster_network_policy_enabled = !local.cluster_output_network_policy_enabled - cluster_http_load_balancing_enabled = !local.cluster_output_http_load_balancing_enabled - cluster_horizontal_pod_autoscaling_enabled = !local.cluster_output_horizontal_pod_autoscaling_enabled - workload_identity_enabled = !(var.identity_namespace == null || var.identity_namespace == "null") + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + {% endif %} + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] diff --git a/autogen/main/masq.tf.tmpl b/autogen/main/masq.tf.tmpl index 8eaa6689d6..28d9b602a1 100644 --- a/autogen/main/masq.tf.tmpl +++ b/autogen/main/masq.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,8 @@ EOF depends_on = [ google_container_cluster.primary, + {% if autopilot_cluster != true %} google_container_node_pool.pools, + {% endif %} ] } diff --git a/autogen/main/networks.tf b/autogen/main/networks.tf index 9a47269879..2d526cdebe 100644 --- a/autogen/main/networks.tf +++ b/autogen/main/networks.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/autogen/main/outputs.tf.tmpl b/autogen/main/outputs.tf.tmpl index decba152d1..951c48ae8b 100644 --- a/autogen/main/outputs.tf.tmpl +++ b/autogen/main/outputs.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -58,7 +58,9 @@ output "endpoint" { * to be up. */ google_container_cluster.primary, + {% if autopilot_cluster != true %} google_container_node_pool.pools, + {% endif %} ] } @@ -93,10 +95,12 @@ output "ca_certificate" { value = local.cluster_ca_certificate } +{% if autopilot_cluster != true %} output "network_policy_enabled" { description = "Whether network policy enabled" value = local.cluster_network_policy_enabled } +{% endif %} output "http_load_balancing_enabled" { description = "Whether http load balancing enabled" @@ -108,6 +112,7 @@ output "horizontal_pod_autoscaling_enabled" { value = local.cluster_horizontal_pod_autoscaling_enabled } +{% if autopilot_cluster != true %} output "node_pools_names" { description = "List of node pools names" value = local.cluster_node_pools_names @@ -117,16 +122,19 @@ output "node_pools_versions" { description = "Node pool versions by node pool name" value = local.cluster_node_pools_versions } +{% endif %} output "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`." value = local.service_account } +{% if autopilot_cluster != true %} output "instance_group_urls" { description = "List of GKE generated instance groups" value = distinct(flatten([for np in google_container_node_pool.pools : np.managed_instance_group_urls])) } +{% endif %} output "release_channel" { description = "The release channel of this cluster" diff --git a/autogen/main/sa.tf.tmpl b/autogen/main/sa.tf.tmpl index 28074556ef..16aaf5287f 100644 --- a/autogen/main/sa.tf.tmpl +++ b/autogen/main/sa.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 94c2216dc9..5b123a0ea3 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -150,6 +150,7 @@ variable "ip_range_services" { description = "The _name_ of the secondary subnet range to use for services" } +{% if autopilot_cluster != true %} variable "initial_node_count" { type = number description = "The number of nodes to create in this cluster's default node pool." @@ -161,6 +162,7 @@ variable "remove_default_node_pool" { description = "Remove default node pool while setting up the cluster" default = false } +{% endif %} variable "disable_legacy_metadata_endpoints" { type = bool @@ -168,6 +170,7 @@ variable "disable_legacy_metadata_endpoints" { default = true } +{% if autopilot_cluster != true %} variable "node_pools" { type = list(map(string)) description = "List of maps containing node pools" @@ -213,6 +216,7 @@ variable "node_pools_linux_node_configs_sysctls" { } } {% endif %} +{% endif %} variable "resource_usage_export_dataset_id" { type = string @@ -266,6 +270,7 @@ variable "cluster_autoscaling" { description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } +{% if autopilot_cluster != true %} variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" @@ -298,6 +303,7 @@ variable "node_pools_oauth_scopes" { default-node-pool = [] } } +{% endif %} variable "stub_domains" { type = map(list(string)) @@ -476,13 +482,21 @@ variable "istio_auth" { variable "dns_cache" { type = bool description = "(Beta) The status of the NodeLocal DNSCache addon." + {% if autopilot_cluster == true %} + default = true + {% else %} default = false + {% endif %} } variable "gce_pd_csi_driver" { type = bool description = "(Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + {% if autopilot_cluster == true %} + default = true + {% else %} default = false + {% endif %} } variable "kalm_config" { diff --git a/autogen/main/variables_defaults.tf b/autogen/main/variables_defaults.tf index 977ebf6a1b..dd30642eed 100644 --- a/autogen/main/variables_defaults.tf +++ b/autogen/main/variables_defaults.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ # Setup dynamic default values for variables which can't be setup using # the standard terraform "variable default" functionality +{% if autopilot_cluster != true %} locals { node_pools_labels = merge( { all = {} }, @@ -82,3 +83,4 @@ locals { ) {% endif %} } +{% endif %} diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl index 47ebc1da69..dfbfb59a46 100644 --- a/autogen/main/versions.tf.tmpl +++ b/autogen/main/versions.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/autogen/safer-cluster/main.tf.tmpl b/autogen/safer-cluster/main.tf.tmpl index 925b7b0408..79d6807d02 100644 --- a/autogen/safer-cluster/main.tf.tmpl +++ b/autogen/safer-cluster/main.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/autogen/safer-cluster/outputs.tf.tmpl b/autogen/safer-cluster/outputs.tf.tmpl index cf82ca0274..e84d0aad38 100644 --- a/autogen/safer-cluster/outputs.tf.tmpl +++ b/autogen/safer-cluster/outputs.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl index 7dd286b2a0..621016886b 100644 --- a/autogen/safer-cluster/variables.tf.tmpl +++ b/autogen/safer-cluster/variables.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/autogen/safer-cluster/versions.tf.tmpl b/autogen/safer-cluster/versions.tf.tmpl index 6353e28f16..b7c17886b8 100644 --- a/autogen/safer-cluster/versions.tf.tmpl +++ b/autogen/safer-cluster/versions.tf.tmpl @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/autogen_modules.json b/autogen_modules.json index 7944554a64..babb5535b4 100644 --- a/autogen_modules.json +++ b/autogen_modules.json @@ -23,6 +23,16 @@ "beta_cluster": true } }, + { + "template_folder": "./autogen/main", + "path": "./modules/beta-autopilot-private-cluster", + "options": { + "module_path": "//modules/beta-autopilot-private-cluster", + "private_cluster": true, + "autopilot_cluster": true, + "beta_cluster": true + } + }, { "template_folder": "./autogen/main", "path": "./modules/private-cluster-update-variant", @@ -51,6 +61,16 @@ "beta_cluster": true } }, + { + "template_folder": "./autogen/main", + "path": "./modules/beta-autopilot-public-cluster", + "options": { + "module_path": "//modules/beta-autopilot-public-cluster", + "private_cluster": false, + "autopilot_cluster": true, + "beta_cluster": true + } + }, { "template_folder": "./autogen/main", "path": "./modules/beta-public-cluster-update-variant", diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index fb44cefe5e..929221c20f 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -329,6 +329,36 @@ steps: - verify simple-zonal-with-asm-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-zonal-with-asm-local'] +- id: converge simple-autopilot-private-local + waitFor: + - create all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-autopilot-private-local'] +- id: verify simple-autopilot-private-local + waitFor: + - converge simple-autopilot-private-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-autopilot-private-local'] +- id: destroy simple-autopilot-private-local + waitFor: + - verify simple-autopilot-private-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-autopilot-private-local'] +- id: converge simple-autopilot-public-local + waitFor: + - create all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-autopilot-public-local'] +- id: verify simple-autopilot-public-local + waitFor: + - converge simple-autopilot-public-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-autopilot-public-local'] +- id: destroy simple-autopilot-public-local + waitFor: + - verify simple-autopilot-public-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-autopilot-public-local'] tags: - 'ci' - 'integration' diff --git a/cluster.tf b/cluster.tf index ba1f0dd318..fdfa8d572b 100644 --- a/cluster.tf +++ b/cluster.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,6 @@ resource "google_container_cluster" "primary" { node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = "projects/${local.network_project_id}/global/networks/${var.network}" - dynamic "network_policy" { for_each = local.cluster_network_policy @@ -55,7 +54,6 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service - cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -75,13 +73,10 @@ resource "google_container_cluster" "primary" { } } } - vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling } - - default_max_pods_per_node = var.default_max_pods_per_node - + default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization dynamic "master_authorized_networks_config" { @@ -111,7 +106,6 @@ resource "google_container_cluster" "primary" { horizontal_pod_autoscaling { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -139,7 +133,6 @@ resource "google_container_cluster" "primary" { update = "45m" delete = "45m" } - node_pool { name = "default-pool" initial_node_count = var.initial_node_count @@ -218,9 +211,7 @@ resource "google_container_cluster" "primary" { security_group = authenticator_groups_config.value.security_group } } - } - /****************************************** Create Container Cluster node pools *****************************************/ diff --git a/dns.tf b/dns.tf index 5dd9a8ee24..1a4c059a30 100644 --- a/dns.tf +++ b/dns.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/acm-terraform-blog-part1/terraform/terraform.tfvars b/examples/acm-terraform-blog-part1/terraform/terraform.tfvars index 12dca13ee2..a9c83be15f 100644 --- a/examples/acm-terraform-blog-part1/terraform/terraform.tfvars +++ b/examples/acm-terraform-blog-part1/terraform/terraform.tfvars @@ -14,8 +14,8 @@ * limitations under the License. */ -region = "us-central1" -zone = "us-central1-c" -sync_repo = "https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git" +region = "us-central1" +zone = "us-central1-c" +sync_repo = "https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git" sync_branch = "master" -policy_dir = "examples/acm-terraform-blog-part1/config-root" +policy_dir = "examples/acm-terraform-blog-part1/config-root" diff --git a/examples/acm-terraform-blog-part2/terraform/terraform.tfvars b/examples/acm-terraform-blog-part2/terraform/terraform.tfvars index 5c0aec0b3f..c00675be80 100644 --- a/examples/acm-terraform-blog-part2/terraform/terraform.tfvars +++ b/examples/acm-terraform-blog-part2/terraform/terraform.tfvars @@ -14,8 +14,8 @@ * limitations under the License. */ -region = "us-central1" -zone = "us-central1-c" -sync_repo = "https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git" +region = "us-central1" +zone = "us-central1-c" +sync_repo = "https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git" sync_branch = "master" -policy_dir = "examples/acm-terraform-blog-part2/config-root" +policy_dir = "examples/acm-terraform-blog-part2/config-root" diff --git a/examples/acm-terraform-blog-part3/terraform/terraform.tfvars b/examples/acm-terraform-blog-part3/terraform/terraform.tfvars index df55548fb3..3a52e3a2ca 100644 --- a/examples/acm-terraform-blog-part3/terraform/terraform.tfvars +++ b/examples/acm-terraform-blog-part3/terraform/terraform.tfvars @@ -14,8 +14,8 @@ * limitations under the License. */ -region = "us-central1" -zone = "us-central1-c" -sync_repo = "https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git" +region = "us-central1" +zone = "us-central1-c" +sync_repo = "https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git" sync_branch = "master" -policy_dir = "examples/acm-terraform-blog-part3/config-root" +policy_dir = "examples/acm-terraform-blog-part3/config-root" diff --git a/examples/simple_autopilot_private/README.md b/examples/simple_autopilot_private/README.md new file mode 100644 index 0000000000..ec82330f6a --- /dev/null +++ b/examples/simple_autopilot_private/README.md @@ -0,0 +1,35 @@ +# Simple Regional Autopilot Cluster + +This example illustrates how to create a simple autopilot cluster with beta features. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| region | The region the cluster in | `string` | `"us-central1"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | The cluster ca certificate (base64 encoded) | +| cluster\_name | Cluster name | +| kubernetes\_endpoint | The cluster endpoint | +| location | n/a | +| master\_kubernetes\_version | Kubernetes version of the master | +| network\_name | The name of the VPC being created | +| project\_id | The project ID the cluster is in | +| region | The region in which the cluster resides | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnet\_names | The names of the subnet being created | +| zones | List of zones in which the cluster resides | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_autopilot_private/main.tf b/examples/simple_autopilot_private/main.tf new file mode 100644 index 0000000000..ffbcd2e10c --- /dev/null +++ b/examples/simple_autopilot_private/main.tf @@ -0,0 +1,58 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-autopilot-private" + network_name = "simple-autopilot-private-network" + subnet_name = "simple-autopilot-private-subnet" + master_auth_subnetwork = "simple-autopilot-private-master-subnet" + pods_range_name = "ip-range-pods-simple-autopilot-private" + svc_range_name = "ip-range-svc-simple-autopilot-private" + subnet_names = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]] +} + + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "../../modules/beta-autopilot-private-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster" + regional = true + region = var.region + network = module.gcp-network.network_name + subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name + release_channel = "REGULAR" + enable_vertical_pod_autoscaling = true + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" + + master_authorized_networks = [ + { + cidr_block = "10.60.0.0/17" + display_name = "VPC" + }, + ] +} diff --git a/examples/simple_autopilot_private/network.tf b/examples/simple_autopilot_private/network.tf new file mode 100644 index 0000000000..cc984f1e5c --- /dev/null +++ b/examples/simple_autopilot_private/network.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = ">= 4.0.1, < 5.0.0" + + project_id = var.project_id + network_name = local.network_name + + subnets = [ + { + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" + subnet_region = var.region + }, + { + subnet_name = local.master_auth_subnetwork + subnet_ip = "10.60.0.0/17" + subnet_region = var.region + }, + ] + + secondary_ranges = { + (local.subnet_name) = [ + { + range_name = local.pods_range_name + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = local.svc_range_name + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} diff --git a/examples/simple_autopilot_private/outputs.tf b/examples/simple_autopilot_private/outputs.tf new file mode 100644 index 0000000000..32f80d64c1 --- /dev/null +++ b/examples/simple_autopilot_private/outputs.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "location" { + value = module.gke.location +} + +output "master_kubernetes_version" { + description = "Kubernetes version of the master" + value = module.gke.master_version +} + +output "ca_certificate" { + description = "The cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + +output "network_name" { + description = "The name of the VPC being created" + value = module.gcp-network.network_name +} + +output "subnet_names" { + description = "The names of the subnet being created" + value = module.gcp-network.subnets_names +} + +output "region" { + description = "The region in which the cluster resides" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "project_id" { + description = "The project ID the cluster is in" + value = var.project_id +} diff --git a/examples/simple_autopilot_private/variables.tf b/examples/simple_autopilot_private/variables.tf new file mode 100644 index 0000000000..28734310c5 --- /dev/null +++ b/examples/simple_autopilot_private/variables.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "region" { + description = "The region the cluster in" + default = "us-central1" +} diff --git a/examples/simple_autopilot_private/versions.tf b/examples/simple_autopilot_private/versions.tf new file mode 100644 index 0000000000..210a187481 --- /dev/null +++ b/examples/simple_autopilot_private/versions.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + required_version = ">= 0.13" +} diff --git a/examples/simple_autopilot_public/README.md b/examples/simple_autopilot_public/README.md new file mode 100644 index 0000000000..0e149933b1 --- /dev/null +++ b/examples/simple_autopilot_public/README.md @@ -0,0 +1,35 @@ +# Simple Regional Cluster + +This example illustrates how to create a simple cluster with beta features. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| region | The region the cluster in | `string` | `"us-central1"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | The cluster ca certificate (base64 encoded) | +| cluster\_name | Cluster name | +| kubernetes\_endpoint | The cluster endpoint | +| location | n/a | +| master\_kubernetes\_version | Kubernetes version of the master | +| network\_name | The name of the VPC being created | +| project\_id | The project ID the cluster is in | +| region | The region in which the cluster resides | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnet\_names | The names of the subnet being created | +| zones | List of zones in which the cluster resides | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_autopilot_public/main.tf b/examples/simple_autopilot_public/main.tf new file mode 100644 index 0000000000..fd55d5ff7a --- /dev/null +++ b/examples/simple_autopilot_public/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-autopilot-public" + network_name = "simple-autopilot-public-network" + subnet_name = "simple-autopilot-public-subnet" + master_auth_subnetwork = "simple-autopilot-public-master-subnet" + pods_range_name = "ip-range-pods-simple-autopilot-public" + svc_range_name = "ip-range-svc-simple-autopilot-public" + subnet_names = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]] +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "../../modules/beta-autopilot-public-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster" + regional = true + region = var.region + network = module.gcp-network.network_name + subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name + release_channel = "REGULAR" + enable_vertical_pod_autoscaling = true +} diff --git a/examples/simple_autopilot_public/network.tf b/examples/simple_autopilot_public/network.tf new file mode 100644 index 0000000000..cc984f1e5c --- /dev/null +++ b/examples/simple_autopilot_public/network.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = ">= 4.0.1, < 5.0.0" + + project_id = var.project_id + network_name = local.network_name + + subnets = [ + { + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" + subnet_region = var.region + }, + { + subnet_name = local.master_auth_subnetwork + subnet_ip = "10.60.0.0/17" + subnet_region = var.region + }, + ] + + secondary_ranges = { + (local.subnet_name) = [ + { + range_name = local.pods_range_name + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = local.svc_range_name + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} diff --git a/examples/simple_autopilot_public/outputs.tf b/examples/simple_autopilot_public/outputs.tf new file mode 100644 index 0000000000..32f80d64c1 --- /dev/null +++ b/examples/simple_autopilot_public/outputs.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "location" { + value = module.gke.location +} + +output "master_kubernetes_version" { + description = "Kubernetes version of the master" + value = module.gke.master_version +} + +output "ca_certificate" { + description = "The cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + +output "network_name" { + description = "The name of the VPC being created" + value = module.gcp-network.network_name +} + +output "subnet_names" { + description = "The names of the subnet being created" + value = module.gcp-network.subnets_names +} + +output "region" { + description = "The region in which the cluster resides" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "project_id" { + description = "The project ID the cluster is in" + value = var.project_id +} diff --git a/examples/simple_autopilot_public/variables.tf b/examples/simple_autopilot_public/variables.tf new file mode 100644 index 0000000000..28734310c5 --- /dev/null +++ b/examples/simple_autopilot_public/variables.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "region" { + description = "The region the cluster in" + default = "us-central1" +} diff --git a/examples/simple_autopilot_public/versions.tf b/examples/simple_autopilot_public/versions.tf new file mode 100644 index 0000000000..210a187481 --- /dev/null +++ b/examples/simple_autopilot_public/versions.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + required_version = ">= 0.13" +} diff --git a/firewall.tf b/firewall.tf index 8ac7624d15..8586855e89 100644 --- a/firewall.tf +++ b/firewall.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/main.tf b/main.tf index 34a0fc323d..24297b180f 100644 --- a/main.tf +++ b/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,6 @@ locals { master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - // Build a map of maps of node pools from a list of objects node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) @@ -81,7 +80,6 @@ locals { provider = null }] - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group }] diff --git a/masq.tf b/masq.tf index 2c45975992..b356aee251 100644 --- a/masq.tf +++ b/masq.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md new file mode 100644 index 0000000000..f5d916f4d3 --- /dev/null +++ b/modules/beta-autopilot-private-cluster/README.md @@ -0,0 +1,235 @@ +# Terraform Kubernetes Engine Module + +This module handles opinionated Google Cloud Platform Kubernetes Engine cluster creation and configuration with Node Pools, IP MASQ, Network Policy, etc. This particular submodule creates a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters)Beta features are enabled in this submodule. +The resources/services/activations/deletions that this module will create/trigger are: +- Create a GKE cluster with the provided addons +- Create GKE Node Pool(s) with provided configuration and attach to cluster +- Replace the default kube-dns configmap if `stub_domains` are provided +- Activate network policy if `network_policy` is true +- Add `ip-masq-agent` configmap with provided `non_masquerade_cidrs` if `configure_ip_masq` is true + +Sub modules are provided for creating private clusters, beta private clusters, and beta public clusters as well. Beta sub modules allow for the use of various GKE beta features. See the modules directory for the various sub modules. + +## Private Cluster Details +For details on configuring private clusters with this module, check the [troubleshooting guide](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/blob/master/docs/private_clusters.md). + +## Compatibility + +This module is meant for use with Terraform 0.13+ and tested using Terraform 1.0+. +If you find incompatibilities using Terraform `>=0.13`, please open an issue. + +If you haven't [upgraded][terraform-0.13-upgrade] and need a Terraform +0.12.x-compatible version of this module, the last released version +intended for Terraform 0.12.x is [12.3.0]. + +## Usage +There are multiple examples included in the [examples](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/examples) folder but simple usage is as follows: + +```hcl +# google_client_config and kubernetes provider must be explicitly specified like the following. +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-private-cluster" + project_id = "" + name = "gke-test-1" + region = "us-central1" + zones = ["us-central1-a", "us-central1-b", "us-central1-f"] + network = "vpc-01" + subnetwork = "us-central1-01" + ip_range_pods = "us-central1-01-gke-01-pods" + ip_range_services = "us-central1-01-gke-01-services" + horizontal_pod_autoscaling = true + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "10.0.0.0/28" + enable_autopilot = true + +} +``` + + +Then perform the following commands on the root folder: + +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | +| config\_connector | (Beta) Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `true` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_identity\_service | Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcloud\_upgrade | Whether to upgrade gcloud at runtime | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| impersonate\_service\_account | An optional service account to impersonate for gcloud commands. If this service account is not specified, the module will use Application Default Credentials. | `string` | `""` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_global\_access\_enabled | (Beta) Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | `string` | `null` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | +| zones | List of zones in which the cluster resides | + + + +## Requirements + +Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: + +1. Terraform and kubectl are [installed](#software-dependencies) on the machine where Terraform is executed. +2. The Service Account you execute the module with has the right [permissions](#configure-a-service-account). +3. The Compute Engine and Kubernetes Engine APIs are [active](#enable-apis) on the project you will launch the cluster in. +4. If you are using a Shared VPC, the APIs must also be activated on the Shared VPC host project and your service account needs the proper permissions there. + +The [project factory](https://github.com/terraform-google-modules/terraform-google-project-factory) can be used to provision projects with the correct APIs active and the necessary Shared VPC connections. + +### Software Dependencies +#### Kubectl +- [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x +#### Terraform and Plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.12 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +#### gcloud +Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. +See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. + +### Configure a Service Account +In order to execute this module you must have a Service Account with the +following project roles: +- roles/compute.viewer +- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`) +- roles/container.clusterAdmin +- roles/container.developer +- roles/iam.serviceAccountAdmin +- roles/iam.serviceAccountUser +- roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) + +Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_ids` projects: +- roles/resourcemanager.projectIamAdmin + +### Enable APIs +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Compute Engine API - compute.googleapis.com +- Kubernetes Engine API - container.googleapis.com + +[terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta +[12.3.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/12.3.0 +[terraform-0.13-upgrade]: https://www.terraform.io/upgrade-guides/0-13.html diff --git a/modules/beta-autopilot-private-cluster/auth.tf b/modules/beta-autopilot-private-cluster/auth.tf new file mode 100644 index 0000000000..8e582145f7 --- /dev/null +++ b/modules/beta-autopilot-private-cluster/auth.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +/****************************************** + Retrieve authentication token + *****************************************/ +data "google_client_config" "default" { + provider = google-beta +} diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf new file mode 100644 index 0000000000..fb0237755f --- /dev/null +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -0,0 +1,176 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +/****************************************** + Create Container Cluster + *****************************************/ +resource "google_container_cluster" "primary" { + provider = google-beta + + name = var.name + description = var.description + project = var.project_id + resource_labels = var.cluster_resource_labels + + location = local.location + node_locations = local.node_locations + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = "projects/${local.network_project_id}/global/networks/${var.network}" + + dynamic "release_channel" { + for_each = local.release_channel + + content { + channel = release_channel.value.channel + } + } + dynamic "confidential_nodes" { + for_each = local.confidential_node_config + content { + enabled = confidential_nodes.value.enabled + } + } + + subnetwork = "projects/${local.network_project_id}/regions/${local.region}/subnetworks/${var.subnetwork}" + + default_snat_status { + disabled = var.disable_default_snat + } + min_master_version = var.release_channel != null ? null : local.master_version + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + vertical_pod_autoscaling { + enabled = var.enable_vertical_pod_autoscaling + } + enable_autopilot = true + dynamic "master_authorized_networks_config" { + for_each = local.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } + + master_auth { + client_certificate_config { + issue_client_certificate = var.issue_client_certificate + } + } + + addons_config { + http_load_balancing { + disabled = !var.http_load_balancing + } + + horizontal_pod_autoscaling { + disabled = !var.horizontal_pod_autoscaling + } + } + + datapath_provider = var.datapath_provider + + networking_mode = "VPC_NATIVE" + ip_allocation_policy { + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services + } + + maintenance_policy { + dynamic "recurring_window" { + for_each = local.cluster_maintenance_window_is_recurring + content { + start_time = var.maintenance_start_time + end_time = var.maintenance_end_time + recurrence = var.maintenance_recurrence + } + } + + dynamic "daily_maintenance_window" { + for_each = local.cluster_maintenance_window_is_daily + content { + start_time = var.maintenance_start_time + } + } + + dynamic "maintenance_exclusion" { + for_each = var.maintenance_exclusions + content { + exclusion_name = maintenance_exclusion.value.name + start_time = maintenance_exclusion.value.start_time + end_time = maintenance_exclusion.value.end_time + } + } + } + + + timeouts { + create = "45m" + update = "45m" + delete = "45m" + } + + dynamic "resource_usage_export_config" { + for_each = var.resource_usage_export_dataset_id != "" ? [{ + enable_network_egress_metering = var.enable_network_egress_export + enable_resource_consumption_metering = var.enable_resource_consumption_export + dataset_id = var.resource_usage_export_dataset_id + }] : [] + + content { + enable_network_egress_metering = resource_usage_export_config.value.enable_network_egress_metering + enable_resource_consumption_metering = resource_usage_export_config.value.enable_resource_consumption_metering + bigquery_destination { + dataset_id = resource_usage_export_config.value.dataset_id + } + } + } + + dynamic "private_cluster_config" { + for_each = var.enable_private_nodes ? [{ + enable_private_nodes = var.enable_private_nodes, + enable_private_endpoint = var.enable_private_endpoint + master_ipv4_cidr_block = var.master_ipv4_cidr_block + }] : [] + + content { + enable_private_endpoint = private_cluster_config.value.enable_private_endpoint + enable_private_nodes = private_cluster_config.value.enable_private_nodes + master_ipv4_cidr_block = private_cluster_config.value.master_ipv4_cidr_block + dynamic "master_global_access_config" { + for_each = var.master_global_access_enabled ? [var.master_global_access_enabled] : [] + content { + enabled = master_global_access_config.value + } + } + } + } + + + notification_config { + pubsub { + enabled = var.notification_config_topic != "" ? true : false + topic = var.notification_config_topic + } + } +} diff --git a/modules/beta-autopilot-private-cluster/dns.tf b/modules/beta-autopilot-private-cluster/dns.tf new file mode 100644 index 0000000000..07f05d1327 --- /dev/null +++ b/modules/beta-autopilot-private-cluster/dns.tf @@ -0,0 +1,119 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +/****************************************** + Delete default kube-dns configmap + *****************************************/ +module "gcloud_delete_default_kube_dns_configmap" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + enabled = (local.custom_kube_dns_config || local.upstream_nameservers_config) && !var.skip_provisioners + cluster_name = google_container_cluster.primary.name + cluster_location = google_container_cluster.primary.location + project_id = var.project_id + upgrade = var.gcloud_upgrade + impersonate_service_account = var.impersonate_service_account + + kubectl_create_command = "${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" + kubectl_destroy_command = "" + + module_depends_on = concat( + [google_container_cluster.primary.master_version], + ) +} + +/****************************************** + Create kube-dns confimap + *****************************************/ +resource "kubernetes_config_map" "kube-dns" { + count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 + + metadata { + name = "kube-dns" + namespace = "kube-system" + + labels = { + maintained_by = "terraform" + } + } + + data = { + stubDomains = <--all INGRESS + firewall rule created by GKE but for EGRESS + + Required for clusters when VPCs enforce + a default-deny egress rule + *****************************************/ +resource "google_compute_firewall" "intra_egress" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress" + description = "Managed by terraform gke module: Allow pods to communicate with each other and the master" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [ + local.cluster_endpoint_for_nodes, + local.cluster_subnet_cidr, + local.cluster_alias_ranges_cidr[var.ip_range_pods], + ] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + +} + + +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + +} + + +/****************************************** + Allow GKE master to hit non 443 ports for + Webhooks/Admission Controllers + + https://github.com/kubernetes/kubernetes/issues/79739 + *****************************************/ +resource "google_compute_firewall" "master_webhooks" { + count = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks" + description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + source_tags = [] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = var.firewall_inbound_ports + } + + +} + + +/****************************************** + Create shadow firewall rules to capture the + traffic flow between the managed firewall rules + *****************************************/ +resource "google_compute_firewall" "shadow_allow_pods" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all" + description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + target_tags = [local.cluster_network_tag] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_master" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10250", "443"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_nodes" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms" + description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_subnet_cidr] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} diff --git a/modules/beta-autopilot-private-cluster/main.tf b/modules/beta-autopilot-private-cluster/main.tf new file mode 100644 index 0000000000..90dfa0b5c1 --- /dev/null +++ b/modules/beta-autopilot-private-cluster/main.tf @@ -0,0 +1,183 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +/****************************************** + Get available zones in region + *****************************************/ +data "google_compute_zones" "available" { + provider = google-beta + + project = var.project_id + region = local.region +} + +resource "random_shuffle" "available_zones" { + input = data.google_compute_zones.available.names + result_count = 3 +} + +locals { + // ID of the cluster + cluster_id = google_container_cluster.primary.id + + // location + location = var.regional ? var.region : var.zones[0] + region = var.regional ? var.region : join("-", slice(split("-", var.zones[0]), 0, 2)) + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // Kubernetes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }], var.cluster_autoscaling.gpu_resources) : [] + + + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 + upstream_nameservers_config = length(var.upstream_nameservers) > 0 + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + zone_count = length(var.zones) + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + // When a release channel is used, node auto-upgrade are enabled and cannot be disabled. + default_auto_upgrade = var.regional || var.release_channel != null ? true : false + + cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null + cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + + cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { + load_balancer_type = var.cloudrun_load_balancer_type + } : {} + cluster_cloudrun_config = var.cloudrun ? [ + merge( + { + disabled = false + }, + local.cluster_cloudrun_config_load_balancer_config + ) + ] : [] + + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + + + cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ + security_group = var.authenticator_security_group + }] + + // legacy mappings https://github.com/hashicorp/terraform-provider-google/pull/10238 + old_node_metadata_config_mapping = { GKE_METADATA_SERVER = "GKE_METADATA", GCE_METADATA = "EXPOSE" } + + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + mode = lookup(local.old_node_metadata_config_mapping, var.node_metadata, var.node_metadata) + }] + + cluster_output_name = google_container_cluster.primary.name + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones + + cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint + cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint_for_nodes = var.master_ipv4_cidr_block + + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + + # BETA features + cluster_output_istio_disabled = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config.0.enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling.0.enabled : false + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config.0.enabled : false + + # /BETA features + + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + + + cluster_master_auth_list_layer1 = local.cluster_output_master_auth + cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] + cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] + + cluster_location = google_container_cluster.primary.location + cluster_region = var.regional ? var.region : join("-", slice(split("-", local.cluster_location), 0, 2)) + cluster_zones = sort(local.cluster_output_zones) + + cluster_name = local.cluster_output_name + cluster_network_tag = "gke-${var.name}" + cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_http_load_balancing_enabled = !local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = !local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = !(var.identity_namespace == null || var.identity_namespace == "null") + cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ + workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace + }] + # BETA features + cluster_istio_enabled = !local.cluster_output_istio_disabled + cluster_cloudrun_enabled = var.cloudrun + cluster_dns_cache_enabled = var.dns_cache + cluster_telemetry_type_is_set = var.cluster_telemetry_type != null + cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + + # /BETA features + + cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] + cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] +} + +/****************************************** + Get available container engine versions + *****************************************/ +data "google_container_engine_versions" "region" { + location = local.location + project = var.project_id +} + +data "google_container_engine_versions" "zone" { + // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error + // + // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. + // + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id +} diff --git a/modules/beta-autopilot-private-cluster/masq.tf b/modules/beta-autopilot-private-cluster/masq.tf new file mode 100644 index 0000000000..65d3cc83c3 --- /dev/null +++ b/modules/beta-autopilot-private-cluster/masq.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +/****************************************** + Create ip-masq-agent confimap + *****************************************/ +resource "kubernetes_config_map" "ip-masq-agent" { + count = var.configure_ip_masq ? 1 : 0 + + metadata { + name = "ip-masq-agent" + namespace = "kube-system" + + labels = { + maintained_by = "terraform" + } + } + + data = { + config = < 0 ? local.cluster_workload_identity_config[0].workload_pool : null + depends_on = [ + google_container_cluster.primary + ] +} + +output "master_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the hosted master network" + value = var.master_ipv4_cidr_block +} + +output "peering_name" { + description = "The name of the peering between this cluster and the Google owned VPC." + value = local.cluster_peering_name +} + +output "istio_enabled" { + description = "Whether Istio is enabled" + value = local.cluster_istio_enabled +} + +output "cloudrun_enabled" { + description = "Whether CloudRun enabled" + value = local.cluster_cloudrun_enabled +} + +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + +output "pod_security_policy_enabled" { + description = "Whether pod security policy is enabled" + value = local.cluster_pod_security_policy_enabled +} + +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled +} + +output "vertical_pod_autoscaling_enabled" { + description = "Whether veritical pod autoscaling is enabled" + value = local.cluster_vertical_pod_autoscaling_enabled +} + +output "identity_service_enabled" { + description = "Whether Identity Service is enabled" + value = local.cluster_pod_security_policy_enabled +} + +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} diff --git a/modules/beta-autopilot-private-cluster/sa.tf b/modules/beta-autopilot-private-cluster/sa.tf new file mode 100644 index 0000000000..eb63753625 --- /dev/null +++ b/modules/beta-autopilot-private-cluster/sa.tf @@ -0,0 +1,86 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +locals { + service_account_list = compact( + concat( + google_service_account.cluster_service_account.*.email, + ["dummy"], + ), + ) + // if user set var.service_account it will be used even if var.create_service_account==true, so service account will be created but not used + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account + + registry_projects_list = length(var.registry_project_ids) == 0 ? [var.project_id] : var.registry_project_ids +} + +resource "random_string" "cluster_service_account_suffix" { + upper = false + lower = true + special = false + length = 4 +} + +resource "google_service_account" "cluster_service_account" { + count = var.create_service_account ? 1 : 0 + project = var.project_id + account_id = "tf-gke-${substr(var.name, 0, min(15, length(var.name)))}-${random_string.cluster_service_account_suffix.result}" + display_name = "Terraform-managed service account for cluster ${var.name}" +} + +resource "google_project_iam_member" "cluster_service_account-log_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/logging.logWriter" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} + +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_project_iam_member.cluster_service_account-log_writer[0].project + role = "roles/monitoring.metricWriter" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} + +resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" { + count = var.create_service_account ? 1 : 0 + project = google_project_iam_member.cluster_service_account-metric_writer[0].project + role = "roles/monitoring.viewer" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_project_iam_member.cluster_service_account-monitoring_viewer[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} + +resource "google_project_iam_member" "cluster_service_account-gcr" { + for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] + project = each.key + role = "roles/storage.objectViewer" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} + +resource "google_project_iam_member" "cluster_service_account-artifact-registry" { + for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] + project = each.key + role = "roles/artifactregistry.reader" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} diff --git a/modules/beta-autopilot-private-cluster/scripts/delete-default-resource.sh b/modules/beta-autopilot-private-cluster/scripts/delete-default-resource.sh new file mode 100755 index 0000000000..3c3c3b212c --- /dev/null +++ b/modules/beta-autopilot-private-cluster/scripts/delete-default-resource.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -ne 3 ]; then + >&2 echo "3 arguments expected. Exiting." + exit 1 +fi + +RESOURCE_NAMESPACE=$1 +RESOURCE_TYPE=$2 +RESOURCE_NAME=$3 + +RESOURCE_LIST=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" || exit 1) + +# Delete requested resource +if [[ $RESOURCE_LIST = *"${RESOURCE_NAME}"* ]]; then + RESOURCE_MAINTAINED_LABEL=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" "${RESOURCE_NAME}" -o=jsonpath='{.metadata.labels.maintained_by}') + if [[ $RESOURCE_MAINTAINED_LABEL = "terraform" ]]; then + echo "Terraform maintained ${RESOURCE_NAME} ${RESOURCE_TYPE} appears to have already been created in ${RESOURCE_NAMESPACE} namespace" + else + echo "Deleting default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" + kubectl -n "${RESOURCE_NAMESPACE}" delete "${RESOURCE_TYPE}" "${RESOURCE_NAME}" + fi +else + echo "No default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" +fi diff --git a/modules/beta-autopilot-private-cluster/scripts/kubectl_wrapper.sh b/modules/beta-autopilot-private-cluster/scripts/kubectl_wrapper.sh new file mode 100755 index 0000000000..a65c30b583 --- /dev/null +++ b/modules/beta-autopilot-private-cluster/scripts/kubectl_wrapper.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +HOST=$1 +TOKEN=$2 +CA_CERTIFICATE=$3 + +shift 3 + +RANDOM_ID="${RANDOM}_${RANDOM}" +export TMPDIR="/tmp/kubectl_wrapper_${RANDOM_ID}" + +function cleanup { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +mkdir "${TMPDIR}" + +export KUBECONFIG="${TMPDIR}/config" + +# shellcheck disable=SC1117 +base64 --help | grep "\--decode" && B64_ARG="--decode" || B64_ARG="-d" +echo "${CA_CERTIFICATE}" | base64 ${B64_ARG} > "${TMPDIR}/ca_certificate" + +kubectl config set-cluster kubectl-wrapper --server="${HOST}" --certificate-authority="${TMPDIR}/ca_certificate" --embed-certs=true 1>/dev/null +rm -f "${TMPDIR}/ca_certificate" +kubectl config set-context kubectl-wrapper --cluster=kubectl-wrapper --user=kubectl-wrapper --namespace=default 1>/dev/null +kubectl config set-credentials kubectl-wrapper --token="${TOKEN}" 1>/dev/null +kubectl config use-context kubectl-wrapper 1>/dev/null +kubectl version 1>/dev/null + +"$@" diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf new file mode 100644 index 0000000000..0faa8f3f83 --- /dev/null +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -0,0 +1,554 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +variable "project_id" { + type = string + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + type = string + description = "The name of the cluster (required)" +} + +variable "description" { + type = string + description = "The description of the cluster" + default = "" +} + +variable "regional" { + type = bool + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + type = string + description = "The region to host the cluster in (optional if zonal cluster / required if regional)" + default = null +} + +variable "zones" { + type = list(string) + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [] +} + +variable "network" { + type = string + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + type = string + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region." + default = "latest" +} + +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." + default = [] +} + +variable "enable_vertical_pod_autoscaling" { + type = bool + description = "Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it" + default = false +} + +variable "horizontal_pod_autoscaling" { + type = bool + description = "Enable horizontal pod autoscaling addon" + default = true +} + +variable "http_load_balancing" { + type = bool + description = "Enable httpload balancer addon" + default = true +} + +variable "network_policy" { + type = bool + description = "Enable network policy addon" + default = false +} + +variable "network_policy_provider" { + type = string + description = "The network policy provider." + default = "CALICO" +} + +variable "datapath_provider" { + type = string + description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." + default = "DATAPATH_PROVIDER_UNSPECIFIED" +} + +variable "maintenance_start_time" { + type = string + description = "Time window specified for daily or recurring maintenance operations in RFC3339 format" + default = "05:00" +} + +variable "maintenance_exclusions" { + type = list(object({ name = string, start_time = string, end_time = string })) + description = "List of maintenance exclusions. A cluster can have up to three" + default = [] +} + +variable "maintenance_end_time" { + type = string + description = "Time window specified for recurring maintenance operations in RFC3339 format" + default = "" +} + +variable "maintenance_recurrence" { + type = string + description = "Frequency of the recurring maintenance window in RFC5545 format." + default = "" +} + +variable "ip_range_pods" { + type = string + description = "The _name_ of the secondary subnet ip range to use for pods" +} + +variable "ip_range_services" { + type = string + description = "The _name_ of the secondary subnet range to use for services" +} + + +variable "disable_legacy_metadata_endpoints" { + type = bool + description = "Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated." + default = true +} + + +variable "resource_usage_export_dataset_id" { + type = string + description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export." + default = "" +} + +variable "enable_network_egress_export" { + type = bool + description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = false +} + +variable "enable_resource_consumption_export" { + type = bool + description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export." + default = true +} + +variable "enable_kubernetes_alpha" { + type = bool + description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." + default = false +} + +variable "cluster_autoscaling" { + type = object({ + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + }) + default = { + enabled = false + autoscaling_profile = "BALANCED" + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + gpu_resources = [] + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + + +variable "stub_domains" { + type = map(list(string)) + description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" + default = {} +} + +variable "upstream_nameservers" { + type = list(string) + description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" + default = [] +} + +variable "non_masquerade_cidrs" { + type = list(string) + description = "List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading." + default = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] +} + +variable "ip_masq_resync_interval" { + type = string + description = "The interval at which the agent attempts to sync its ConfigMap file from the disk." + default = "60s" +} + +variable "ip_masq_link_local" { + type = bool + description = "Whether to masquerade traffic to the link-local prefix (169.254.0.0/16)." + default = false +} + +variable "configure_ip_masq" { + type = bool + description = "Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server." + default = false +} + +variable "cluster_telemetry_type" { + type = string + description = "Available options include ENABLED, DISABLED, and SYSTEM_ONLY" + default = null +} + +variable "logging_service" { + type = string + description = "The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none" + default = "logging.googleapis.com/kubernetes" +} + +variable "logging_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + +variable "monitoring_service" { + type = string + description = "The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none" + default = "monitoring.googleapis.com/kubernetes" +} + +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + default = [] +} + +variable "create_service_account" { + type = bool + description = "Defines if service account specified to run nodes should be created." + default = true +} + +variable "grant_registry_access" { + type = bool + description = "Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles." + default = false +} + +variable "registry_project_ids" { + type = list(string) + description = "Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects." + default = [] +} + +variable "service_account" { + type = string + description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." + default = "" +} + +variable "issue_client_certificate" { + type = bool + description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!" + default = false +} + +variable "cluster_ipv4_cidr" { + type = string + default = null + description = "The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR." +} + +variable "cluster_resource_labels" { + type = map(string) + description = "The GCE resource labels (a map of key/value pairs) to be applied to the cluster" + default = {} +} + +variable "skip_provisioners" { + type = bool + description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." + default = false +} + +variable "default_max_pods_per_node" { + type = number + description = "The maximum number of pods to schedule per node" + default = 110 +} + +variable "deploy_using_private_endpoint" { + type = bool + description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + default = false +} + +variable "enable_private_endpoint" { + type = bool + description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" + default = false +} + +variable "enable_private_nodes" { + type = bool + description = "(Beta) Whether nodes have internal IP addresses only" + default = false +} + +variable "master_ipv4_cidr_block" { + type = string + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" +} + +variable "master_global_access_enabled" { + type = bool + description = "(Beta) Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint." + + default = true +} + + +variable "istio" { + description = "(Beta) Enable Istio addon" + default = false +} + +variable "istio_auth" { + type = string + description = "(Beta) The authentication type between services in Istio." + default = "AUTH_MUTUAL_TLS" +} + +variable "dns_cache" { + type = bool + description = "(Beta) The status of the NodeLocal DNSCache addon." + default = true +} + +variable "gce_pd_csi_driver" { + type = bool + description = "(Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + default = true +} + +variable "kalm_config" { + type = bool + description = "(Beta) Whether KALM is enabled for this cluster." + default = false +} + +variable "config_connector" { + type = bool + description = "(Beta) Whether ConfigConnector is enabled for this cluster." + default = false +} + +variable "cloudrun" { + description = "(Beta) Enable CloudRun addon" + default = false +} + +variable "cloudrun_load_balancer_type" { + description = "(Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer." + default = "" +} + +variable "enable_pod_security_policy" { + type = bool + description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." + default = false +} + +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." + default = false +} + +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + +variable "enable_identity_service" { + type = bool + description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." + default = false +} + +variable "authenticator_security_group" { + type = string + description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" + default = null +} + +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "GKE_METADATA" + type = string + + validation { + condition = contains(["GKE_METADATA", "GCE_METADATA", "UNSPECIFIED", "GKE_METADATA_SERVER", "EXPOSE"], var.node_metadata) + error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA or UNSPECIFIED." + } +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "identity_namespace" { + description = "The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`)" + type = string + default = "enabled" +} + +variable "release_channel" { + type = string + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." + default = null +} + +variable "enable_shielded_nodes" { + type = bool + description = "Enable Shielded Nodes features on all nodes in this cluster" + default = true +} + +variable "enable_binary_authorization" { + type = bool + description = "Enable BinAuthZ Admission controller" + default = false +} + +variable "add_cluster_firewall_rules" { + type = bool + description = "Create additional firewall rules" + default = false +} + +variable "add_master_webhook_firewall_rules" { + type = bool + description = "Create master_webhook firewall rules for ports defined in `firewall_inbound_ports`" + default = false +} + +variable "firewall_priority" { + type = number + description = "Priority rule for firewall rules" + default = 1000 +} + +variable "firewall_inbound_ports" { + type = list(string) + description = "List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied." + default = ["8443", "9443", "15017"] +} + +variable "gcloud_upgrade" { + type = bool + description = "Whether to upgrade gcloud at runtime" + default = false +} + +variable "add_shadow_firewall_rules" { + type = bool + description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)." + default = false +} + +variable "shadow_firewall_rules_priority" { + type = number + description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." + default = 999 +} + +variable "enable_confidential_nodes" { + type = bool + description = "An optional flag to enable confidential node config." + default = false +} + +variable "disable_default_snat" { + type = bool + description = "Whether to disable the default SNAT to support the private use of public IP addresses" + default = false +} + +variable "impersonate_service_account" { + type = string + description = "An optional service account to impersonate for gcloud commands. If this service account is not specified, the module will use Application Default Credentials." + default = "" +} + +variable "notification_config_topic" { + type = string + description = "The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}." + default = "" +} + +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} diff --git a/modules/beta-autopilot-private-cluster/variables_defaults.tf b/modules/beta-autopilot-private-cluster/variables_defaults.tf new file mode 100644 index 0000000000..c06ea2339c --- /dev/null +++ b/modules/beta-autopilot-private-cluster/variables_defaults.tf @@ -0,0 +1,21 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + diff --git a/modules/beta-autopilot-private-cluster/versions.tf b/modules/beta-autopilot-private-cluster/versions.tf new file mode 100644 index 0000000000..dcf4235ae2 --- /dev/null +++ b/modules/beta-autopilot-private-cluster/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +terraform { + required_version = ">=0.13" + + required_providers { + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.6.0, < 5.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.0" + } + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v19.0.0" + } +} diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md new file mode 100644 index 0000000000..8ab379df96 --- /dev/null +++ b/modules/beta-autopilot-public-cluster/README.md @@ -0,0 +1,222 @@ +# Terraform Kubernetes Engine Module + +This module handles opinionated Google Cloud Platform Kubernetes Engine cluster creation and configuration with Node Pools, IP MASQ, Network Policy, etc.Beta features are enabled in this submodule. +The resources/services/activations/deletions that this module will create/trigger are: +- Create a GKE cluster with the provided addons +- Create GKE Node Pool(s) with provided configuration and attach to cluster +- Replace the default kube-dns configmap if `stub_domains` are provided +- Activate network policy if `network_policy` is true +- Add `ip-masq-agent` configmap with provided `non_masquerade_cidrs` if `configure_ip_masq` is true + +Sub modules are provided for creating private clusters, beta private clusters, and beta public clusters as well. Beta sub modules allow for the use of various GKE beta features. See the modules directory for the various sub modules. + +## Compatibility + +This module is meant for use with Terraform 0.13+ and tested using Terraform 1.0+. +If you find incompatibilities using Terraform `>=0.13`, please open an issue. + +If you haven't [upgraded][terraform-0.13-upgrade] and need a Terraform +0.12.x-compatible version of this module, the last released version +intended for Terraform 0.12.x is [12.3.0]. + +## Usage +There are multiple examples included in the [examples](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/examples) folder but simple usage is as follows: + +```hcl +# google_client_config and kubernetes provider must be explicitly specified like the following. +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-public-cluster" + project_id = "" + name = "gke-test-1" + region = "us-central1" + zones = ["us-central1-a", "us-central1-b", "us-central1-f"] + network = "vpc-01" + subnetwork = "us-central1-01" + ip_range_pods = "us-central1-01-gke-01-pods" + ip_range_services = "us-central1-01-gke-01-services" + horizontal_pod_autoscaling = true + enable_autopilot = true + +} +``` + + +Then perform the following commands on the root folder: + +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | +| config\_connector | (Beta) Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `true` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_identity\_service | Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcloud\_upgrade | Whether to upgrade gcloud at runtime | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| impersonate\_service\_account | An optional service account to impersonate for gcloud commands. If this service account is not specified, the module will use Application Default Credentials. | `string` | `""` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | `string` | `null` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | +| zones | List of zones in which the cluster resides | + + + +## Requirements + +Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: + +1. Terraform and kubectl are [installed](#software-dependencies) on the machine where Terraform is executed. +2. The Service Account you execute the module with has the right [permissions](#configure-a-service-account). +3. The Compute Engine and Kubernetes Engine APIs are [active](#enable-apis) on the project you will launch the cluster in. +4. If you are using a Shared VPC, the APIs must also be activated on the Shared VPC host project and your service account needs the proper permissions there. + +The [project factory](https://github.com/terraform-google-modules/terraform-google-project-factory) can be used to provision projects with the correct APIs active and the necessary Shared VPC connections. + +### Software Dependencies +#### Kubectl +- [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x +#### Terraform and Plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.12 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +#### gcloud +Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. +See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. + +### Configure a Service Account +In order to execute this module you must have a Service Account with the +following project roles: +- roles/compute.viewer +- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`) +- roles/container.clusterAdmin +- roles/container.developer +- roles/iam.serviceAccountAdmin +- roles/iam.serviceAccountUser +- roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) + +Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_ids` projects: +- roles/resourcemanager.projectIamAdmin + +### Enable APIs +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Compute Engine API - compute.googleapis.com +- Kubernetes Engine API - container.googleapis.com + +[terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta +[12.3.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/12.3.0 +[terraform-0.13-upgrade]: https://www.terraform.io/upgrade-guides/0-13.html diff --git a/modules/beta-autopilot-public-cluster/auth.tf b/modules/beta-autopilot-public-cluster/auth.tf new file mode 100644 index 0000000000..8e582145f7 --- /dev/null +++ b/modules/beta-autopilot-public-cluster/auth.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +/****************************************** + Retrieve authentication token + *****************************************/ +data "google_client_config" "default" { + provider = google-beta +} diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf new file mode 100644 index 0000000000..f047f4ac1f --- /dev/null +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -0,0 +1,157 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +/****************************************** + Create Container Cluster + *****************************************/ +resource "google_container_cluster" "primary" { + provider = google-beta + + name = var.name + description = var.description + project = var.project_id + resource_labels = var.cluster_resource_labels + + location = local.location + node_locations = local.node_locations + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = "projects/${local.network_project_id}/global/networks/${var.network}" + + dynamic "release_channel" { + for_each = local.release_channel + + content { + channel = release_channel.value.channel + } + } + dynamic "confidential_nodes" { + for_each = local.confidential_node_config + content { + enabled = confidential_nodes.value.enabled + } + } + + subnetwork = "projects/${local.network_project_id}/regions/${local.region}/subnetworks/${var.subnetwork}" + + default_snat_status { + disabled = var.disable_default_snat + } + min_master_version = var.release_channel != null ? null : local.master_version + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + vertical_pod_autoscaling { + enabled = var.enable_vertical_pod_autoscaling + } + enable_autopilot = true + dynamic "master_authorized_networks_config" { + for_each = local.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } + + master_auth { + client_certificate_config { + issue_client_certificate = var.issue_client_certificate + } + } + + addons_config { + http_load_balancing { + disabled = !var.http_load_balancing + } + + horizontal_pod_autoscaling { + disabled = !var.horizontal_pod_autoscaling + } + } + + datapath_provider = var.datapath_provider + + networking_mode = "VPC_NATIVE" + ip_allocation_policy { + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services + } + + maintenance_policy { + dynamic "recurring_window" { + for_each = local.cluster_maintenance_window_is_recurring + content { + start_time = var.maintenance_start_time + end_time = var.maintenance_end_time + recurrence = var.maintenance_recurrence + } + } + + dynamic "daily_maintenance_window" { + for_each = local.cluster_maintenance_window_is_daily + content { + start_time = var.maintenance_start_time + } + } + + dynamic "maintenance_exclusion" { + for_each = var.maintenance_exclusions + content { + exclusion_name = maintenance_exclusion.value.name + start_time = maintenance_exclusion.value.start_time + end_time = maintenance_exclusion.value.end_time + } + } + } + + + timeouts { + create = "45m" + update = "45m" + delete = "45m" + } + + dynamic "resource_usage_export_config" { + for_each = var.resource_usage_export_dataset_id != "" ? [{ + enable_network_egress_metering = var.enable_network_egress_export + enable_resource_consumption_metering = var.enable_resource_consumption_export + dataset_id = var.resource_usage_export_dataset_id + }] : [] + + content { + enable_network_egress_metering = resource_usage_export_config.value.enable_network_egress_metering + enable_resource_consumption_metering = resource_usage_export_config.value.enable_resource_consumption_metering + bigquery_destination { + dataset_id = resource_usage_export_config.value.dataset_id + } + } + } + + + + notification_config { + pubsub { + enabled = var.notification_config_topic != "" ? true : false + topic = var.notification_config_topic + } + } +} diff --git a/modules/beta-autopilot-public-cluster/dns.tf b/modules/beta-autopilot-public-cluster/dns.tf new file mode 100644 index 0000000000..07f05d1327 --- /dev/null +++ b/modules/beta-autopilot-public-cluster/dns.tf @@ -0,0 +1,119 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +/****************************************** + Delete default kube-dns configmap + *****************************************/ +module "gcloud_delete_default_kube_dns_configmap" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + enabled = (local.custom_kube_dns_config || local.upstream_nameservers_config) && !var.skip_provisioners + cluster_name = google_container_cluster.primary.name + cluster_location = google_container_cluster.primary.location + project_id = var.project_id + upgrade = var.gcloud_upgrade + impersonate_service_account = var.impersonate_service_account + + kubectl_create_command = "${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" + kubectl_destroy_command = "" + + module_depends_on = concat( + [google_container_cluster.primary.master_version], + ) +} + +/****************************************** + Create kube-dns confimap + *****************************************/ +resource "kubernetes_config_map" "kube-dns" { + count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 + + metadata { + name = "kube-dns" + namespace = "kube-system" + + labels = { + maintained_by = "terraform" + } + } + + data = { + stubDomains = <--all INGRESS + firewall rule created by GKE but for EGRESS + + Required for clusters when VPCs enforce + a default-deny egress rule + *****************************************/ +resource "google_compute_firewall" "intra_egress" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress" + description = "Managed by terraform gke module: Allow pods to communicate with each other and the master" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [ + local.cluster_endpoint_for_nodes, + local.cluster_subnet_cidr, + local.cluster_alias_ranges_cidr[var.ip_range_pods], + ] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + + +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + + +/****************************************** + Allow GKE master to hit non 443 ports for + Webhooks/Admission Controllers + + https://github.com/kubernetes/kubernetes/issues/79739 + *****************************************/ +resource "google_compute_firewall" "master_webhooks" { + count = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks" + description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + source_tags = [] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = var.firewall_inbound_ports + } + + depends_on = [ + google_container_cluster.primary, + ] + +} + + +/****************************************** + Create shadow firewall rules to capture the + traffic flow between the managed firewall rules + *****************************************/ +resource "google_compute_firewall" "shadow_allow_pods" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all" + description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + target_tags = [local.cluster_network_tag] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_master" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10250", "443"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_nodes" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms" + description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_subnet_cidr] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} diff --git a/modules/beta-autopilot-public-cluster/main.tf b/modules/beta-autopilot-public-cluster/main.tf new file mode 100644 index 0000000000..7ca2e091f0 --- /dev/null +++ b/modules/beta-autopilot-public-cluster/main.tf @@ -0,0 +1,182 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +/****************************************** + Get available zones in region + *****************************************/ +data "google_compute_zones" "available" { + provider = google-beta + + project = var.project_id + region = local.region +} + +resource "random_shuffle" "available_zones" { + input = data.google_compute_zones.available.names + result_count = 3 +} + +locals { + // ID of the cluster + cluster_id = google_container_cluster.primary.id + + // location + location = var.regional ? var.region : var.zones[0] + region = var.regional ? var.region : join("-", slice(split("-", var.zones[0]), 0, 2)) + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // Kubernetes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }], var.cluster_autoscaling.gpu_resources) : [] + + + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 + upstream_nameservers_config = length(var.upstream_nameservers) > 0 + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + zone_count = length(var.zones) + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + // When a release channel is used, node auto-upgrade are enabled and cannot be disabled. + default_auto_upgrade = var.regional || var.release_channel != null ? true : false + + cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null + cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + + cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { + load_balancer_type = var.cloudrun_load_balancer_type + } : {} + cluster_cloudrun_config = var.cloudrun ? [ + merge( + { + disabled = false + }, + local.cluster_cloudrun_config_load_balancer_config + ) + ] : [] + + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + + + cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ + security_group = var.authenticator_security_group + }] + + // legacy mappings https://github.com/hashicorp/terraform-provider-google/pull/10238 + old_node_metadata_config_mapping = { GKE_METADATA_SERVER = "GKE_METADATA", GCE_METADATA = "EXPOSE" } + + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + mode = lookup(local.old_node_metadata_config_mapping, var.node_metadata, var.node_metadata) + }] + + cluster_output_name = google_container_cluster.primary.name + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones + + cluster_endpoint = google_container_cluster.primary.endpoint + cluster_endpoint_for_nodes = "${google_container_cluster.primary.endpoint}/32" + + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + + # BETA features + cluster_output_istio_disabled = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config.0.enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling.0.enabled : false + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config.0.enabled : false + + # /BETA features + + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + + + cluster_master_auth_list_layer1 = local.cluster_output_master_auth + cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] + cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] + + cluster_location = google_container_cluster.primary.location + cluster_region = var.regional ? var.region : join("-", slice(split("-", local.cluster_location), 0, 2)) + cluster_zones = sort(local.cluster_output_zones) + + cluster_name = local.cluster_output_name + cluster_network_tag = "gke-${var.name}" + cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_http_load_balancing_enabled = !local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = !local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = !(var.identity_namespace == null || var.identity_namespace == "null") + cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ + workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace + }] + # BETA features + cluster_istio_enabled = !local.cluster_output_istio_disabled + cluster_cloudrun_enabled = var.cloudrun + cluster_dns_cache_enabled = var.dns_cache + cluster_telemetry_type_is_set = var.cluster_telemetry_type != null + cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + + # /BETA features + + cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] + cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] +} + +/****************************************** + Get available container engine versions + *****************************************/ +data "google_container_engine_versions" "region" { + location = local.location + project = var.project_id +} + +data "google_container_engine_versions" "zone" { + // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error + // + // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. + // + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id +} diff --git a/modules/beta-autopilot-public-cluster/masq.tf b/modules/beta-autopilot-public-cluster/masq.tf new file mode 100644 index 0000000000..65d3cc83c3 --- /dev/null +++ b/modules/beta-autopilot-public-cluster/masq.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +/****************************************** + Create ip-masq-agent confimap + *****************************************/ +resource "kubernetes_config_map" "ip-masq-agent" { + count = var.configure_ip_masq ? 1 : 0 + + metadata { + name = "ip-masq-agent" + namespace = "kube-system" + + labels = { + maintained_by = "terraform" + } + } + + data = { + config = < 0 ? local.cluster_workload_identity_config[0].workload_pool : null + depends_on = [ + google_container_cluster.primary + ] +} + +output "istio_enabled" { + description = "Whether Istio is enabled" + value = local.cluster_istio_enabled +} + +output "cloudrun_enabled" { + description = "Whether CloudRun enabled" + value = local.cluster_cloudrun_enabled +} + +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + +output "pod_security_policy_enabled" { + description = "Whether pod security policy is enabled" + value = local.cluster_pod_security_policy_enabled +} + +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled +} + +output "vertical_pod_autoscaling_enabled" { + description = "Whether veritical pod autoscaling is enabled" + value = local.cluster_vertical_pod_autoscaling_enabled +} + +output "identity_service_enabled" { + description = "Whether Identity Service is enabled" + value = local.cluster_pod_security_policy_enabled +} + +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} diff --git a/modules/beta-autopilot-public-cluster/sa.tf b/modules/beta-autopilot-public-cluster/sa.tf new file mode 100644 index 0000000000..eb63753625 --- /dev/null +++ b/modules/beta-autopilot-public-cluster/sa.tf @@ -0,0 +1,86 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +locals { + service_account_list = compact( + concat( + google_service_account.cluster_service_account.*.email, + ["dummy"], + ), + ) + // if user set var.service_account it will be used even if var.create_service_account==true, so service account will be created but not used + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account + + registry_projects_list = length(var.registry_project_ids) == 0 ? [var.project_id] : var.registry_project_ids +} + +resource "random_string" "cluster_service_account_suffix" { + upper = false + lower = true + special = false + length = 4 +} + +resource "google_service_account" "cluster_service_account" { + count = var.create_service_account ? 1 : 0 + project = var.project_id + account_id = "tf-gke-${substr(var.name, 0, min(15, length(var.name)))}-${random_string.cluster_service_account_suffix.result}" + display_name = "Terraform-managed service account for cluster ${var.name}" +} + +resource "google_project_iam_member" "cluster_service_account-log_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/logging.logWriter" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} + +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_project_iam_member.cluster_service_account-log_writer[0].project + role = "roles/monitoring.metricWriter" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} + +resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" { + count = var.create_service_account ? 1 : 0 + project = google_project_iam_member.cluster_service_account-metric_writer[0].project + role = "roles/monitoring.viewer" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_project_iam_member.cluster_service_account-monitoring_viewer[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} + +resource "google_project_iam_member" "cluster_service_account-gcr" { + for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] + project = each.key + role = "roles/storage.objectViewer" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} + +resource "google_project_iam_member" "cluster_service_account-artifact-registry" { + for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] + project = each.key + role = "roles/artifactregistry.reader" + member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" +} diff --git a/modules/beta-autopilot-public-cluster/scripts/delete-default-resource.sh b/modules/beta-autopilot-public-cluster/scripts/delete-default-resource.sh new file mode 100755 index 0000000000..3c3c3b212c --- /dev/null +++ b/modules/beta-autopilot-public-cluster/scripts/delete-default-resource.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -ne 3 ]; then + >&2 echo "3 arguments expected. Exiting." + exit 1 +fi + +RESOURCE_NAMESPACE=$1 +RESOURCE_TYPE=$2 +RESOURCE_NAME=$3 + +RESOURCE_LIST=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" || exit 1) + +# Delete requested resource +if [[ $RESOURCE_LIST = *"${RESOURCE_NAME}"* ]]; then + RESOURCE_MAINTAINED_LABEL=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" "${RESOURCE_NAME}" -o=jsonpath='{.metadata.labels.maintained_by}') + if [[ $RESOURCE_MAINTAINED_LABEL = "terraform" ]]; then + echo "Terraform maintained ${RESOURCE_NAME} ${RESOURCE_TYPE} appears to have already been created in ${RESOURCE_NAMESPACE} namespace" + else + echo "Deleting default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" + kubectl -n "${RESOURCE_NAMESPACE}" delete "${RESOURCE_TYPE}" "${RESOURCE_NAME}" + fi +else + echo "No default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" +fi diff --git a/modules/beta-autopilot-public-cluster/scripts/kubectl_wrapper.sh b/modules/beta-autopilot-public-cluster/scripts/kubectl_wrapper.sh new file mode 100755 index 0000000000..a65c30b583 --- /dev/null +++ b/modules/beta-autopilot-public-cluster/scripts/kubectl_wrapper.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +HOST=$1 +TOKEN=$2 +CA_CERTIFICATE=$3 + +shift 3 + +RANDOM_ID="${RANDOM}_${RANDOM}" +export TMPDIR="/tmp/kubectl_wrapper_${RANDOM_ID}" + +function cleanup { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +mkdir "${TMPDIR}" + +export KUBECONFIG="${TMPDIR}/config" + +# shellcheck disable=SC1117 +base64 --help | grep "\--decode" && B64_ARG="--decode" || B64_ARG="-d" +echo "${CA_CERTIFICATE}" | base64 ${B64_ARG} > "${TMPDIR}/ca_certificate" + +kubectl config set-cluster kubectl-wrapper --server="${HOST}" --certificate-authority="${TMPDIR}/ca_certificate" --embed-certs=true 1>/dev/null +rm -f "${TMPDIR}/ca_certificate" +kubectl config set-context kubectl-wrapper --cluster=kubectl-wrapper --user=kubectl-wrapper --namespace=default 1>/dev/null +kubectl config set-credentials kubectl-wrapper --token="${TOKEN}" 1>/dev/null +kubectl config use-context kubectl-wrapper 1>/dev/null +kubectl version 1>/dev/null + +"$@" diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf new file mode 100644 index 0000000000..6a4f02e804 --- /dev/null +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -0,0 +1,523 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +variable "project_id" { + type = string + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + type = string + description = "The name of the cluster (required)" +} + +variable "description" { + type = string + description = "The description of the cluster" + default = "" +} + +variable "regional" { + type = bool + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + type = string + description = "The region to host the cluster in (optional if zonal cluster / required if regional)" + default = null +} + +variable "zones" { + type = list(string) + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [] +} + +variable "network" { + type = string + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + type = string + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region." + default = "latest" +} + +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." + default = [] +} + +variable "enable_vertical_pod_autoscaling" { + type = bool + description = "Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it" + default = false +} + +variable "horizontal_pod_autoscaling" { + type = bool + description = "Enable horizontal pod autoscaling addon" + default = true +} + +variable "http_load_balancing" { + type = bool + description = "Enable httpload balancer addon" + default = true +} + +variable "network_policy" { + type = bool + description = "Enable network policy addon" + default = false +} + +variable "network_policy_provider" { + type = string + description = "The network policy provider." + default = "CALICO" +} + +variable "datapath_provider" { + type = string + description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." + default = "DATAPATH_PROVIDER_UNSPECIFIED" +} + +variable "maintenance_start_time" { + type = string + description = "Time window specified for daily or recurring maintenance operations in RFC3339 format" + default = "05:00" +} + +variable "maintenance_exclusions" { + type = list(object({ name = string, start_time = string, end_time = string })) + description = "List of maintenance exclusions. A cluster can have up to three" + default = [] +} + +variable "maintenance_end_time" { + type = string + description = "Time window specified for recurring maintenance operations in RFC3339 format" + default = "" +} + +variable "maintenance_recurrence" { + type = string + description = "Frequency of the recurring maintenance window in RFC5545 format." + default = "" +} + +variable "ip_range_pods" { + type = string + description = "The _name_ of the secondary subnet ip range to use for pods" +} + +variable "ip_range_services" { + type = string + description = "The _name_ of the secondary subnet range to use for services" +} + + +variable "disable_legacy_metadata_endpoints" { + type = bool + description = "Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated." + default = true +} + + +variable "resource_usage_export_dataset_id" { + type = string + description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export." + default = "" +} + +variable "enable_network_egress_export" { + type = bool + description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = false +} + +variable "enable_resource_consumption_export" { + type = bool + description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export." + default = true +} + +variable "enable_kubernetes_alpha" { + type = bool + description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." + default = false +} + +variable "cluster_autoscaling" { + type = object({ + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + }) + default = { + enabled = false + autoscaling_profile = "BALANCED" + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + gpu_resources = [] + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + + +variable "stub_domains" { + type = map(list(string)) + description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" + default = {} +} + +variable "upstream_nameservers" { + type = list(string) + description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" + default = [] +} + +variable "non_masquerade_cidrs" { + type = list(string) + description = "List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading." + default = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] +} + +variable "ip_masq_resync_interval" { + type = string + description = "The interval at which the agent attempts to sync its ConfigMap file from the disk." + default = "60s" +} + +variable "ip_masq_link_local" { + type = bool + description = "Whether to masquerade traffic to the link-local prefix (169.254.0.0/16)." + default = false +} + +variable "configure_ip_masq" { + type = bool + description = "Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server." + default = false +} + +variable "cluster_telemetry_type" { + type = string + description = "Available options include ENABLED, DISABLED, and SYSTEM_ONLY" + default = null +} + +variable "logging_service" { + type = string + description = "The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none" + default = "logging.googleapis.com/kubernetes" +} + +variable "logging_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + +variable "monitoring_service" { + type = string + description = "The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none" + default = "monitoring.googleapis.com/kubernetes" +} + +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + default = [] +} + +variable "create_service_account" { + type = bool + description = "Defines if service account specified to run nodes should be created." + default = true +} + +variable "grant_registry_access" { + type = bool + description = "Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles." + default = false +} + +variable "registry_project_ids" { + type = list(string) + description = "Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects." + default = [] +} + +variable "service_account" { + type = string + description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." + default = "" +} + +variable "issue_client_certificate" { + type = bool + description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!" + default = false +} + +variable "cluster_ipv4_cidr" { + type = string + default = null + description = "The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR." +} + +variable "cluster_resource_labels" { + type = map(string) + description = "The GCE resource labels (a map of key/value pairs) to be applied to the cluster" + default = {} +} + +variable "skip_provisioners" { + type = bool + description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." + default = false +} + +variable "default_max_pods_per_node" { + type = number + description = "The maximum number of pods to schedule per node" + default = 110 +} + + +variable "istio" { + description = "(Beta) Enable Istio addon" + default = false +} + +variable "istio_auth" { + type = string + description = "(Beta) The authentication type between services in Istio." + default = "AUTH_MUTUAL_TLS" +} + +variable "dns_cache" { + type = bool + description = "(Beta) The status of the NodeLocal DNSCache addon." + default = true +} + +variable "gce_pd_csi_driver" { + type = bool + description = "(Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + default = true +} + +variable "kalm_config" { + type = bool + description = "(Beta) Whether KALM is enabled for this cluster." + default = false +} + +variable "config_connector" { + type = bool + description = "(Beta) Whether ConfigConnector is enabled for this cluster." + default = false +} + +variable "cloudrun" { + description = "(Beta) Enable CloudRun addon" + default = false +} + +variable "cloudrun_load_balancer_type" { + description = "(Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer." + default = "" +} + +variable "enable_pod_security_policy" { + type = bool + description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." + default = false +} + +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." + default = false +} + +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + +variable "enable_identity_service" { + type = bool + description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." + default = false +} + +variable "authenticator_security_group" { + type = string + description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" + default = null +} + +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "GKE_METADATA" + type = string + + validation { + condition = contains(["GKE_METADATA", "GCE_METADATA", "UNSPECIFIED", "GKE_METADATA_SERVER", "EXPOSE"], var.node_metadata) + error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA or UNSPECIFIED." + } +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "identity_namespace" { + description = "The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`)" + type = string + default = "enabled" +} + +variable "release_channel" { + type = string + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." + default = null +} + +variable "enable_shielded_nodes" { + type = bool + description = "Enable Shielded Nodes features on all nodes in this cluster" + default = true +} + +variable "enable_binary_authorization" { + type = bool + description = "Enable BinAuthZ Admission controller" + default = false +} + +variable "add_cluster_firewall_rules" { + type = bool + description = "Create additional firewall rules" + default = false +} + +variable "add_master_webhook_firewall_rules" { + type = bool + description = "Create master_webhook firewall rules for ports defined in `firewall_inbound_ports`" + default = false +} + +variable "firewall_priority" { + type = number + description = "Priority rule for firewall rules" + default = 1000 +} + +variable "firewall_inbound_ports" { + type = list(string) + description = "List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied." + default = ["8443", "9443", "15017"] +} + +variable "gcloud_upgrade" { + type = bool + description = "Whether to upgrade gcloud at runtime" + default = false +} + +variable "add_shadow_firewall_rules" { + type = bool + description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)." + default = false +} + +variable "shadow_firewall_rules_priority" { + type = number + description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." + default = 999 +} + +variable "enable_confidential_nodes" { + type = bool + description = "An optional flag to enable confidential node config." + default = false +} + +variable "disable_default_snat" { + type = bool + description = "Whether to disable the default SNAT to support the private use of public IP addresses" + default = false +} + +variable "impersonate_service_account" { + type = string + description = "An optional service account to impersonate for gcloud commands. If this service account is not specified, the module will use Application Default Credentials." + default = "" +} + +variable "notification_config_topic" { + type = string + description = "The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}." + default = "" +} + +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} diff --git a/modules/beta-autopilot-public-cluster/variables_defaults.tf b/modules/beta-autopilot-public-cluster/variables_defaults.tf new file mode 100644 index 0000000000..c06ea2339c --- /dev/null +++ b/modules/beta-autopilot-public-cluster/variables_defaults.tf @@ -0,0 +1,21 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + diff --git a/modules/beta-autopilot-public-cluster/versions.tf b/modules/beta-autopilot-public-cluster/versions.tf new file mode 100644 index 0000000000..e9f3882544 --- /dev/null +++ b/modules/beta-autopilot-public-cluster/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +terraform { + required_version = ">=0.13" + + required_providers { + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.6.0, < 5.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.0" + } + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v19.0.0" + } +} diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 734b5e0f7d..3bb6709467 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -68,14 +68,14 @@ module "gke" { ip_range_pods = "us-central1-01-gke-01-pods" ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false - horizontal_pod_autoscaling = true network_policy = false + horizontal_pod_autoscaling = true enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" - istio = true - cloudrun = true - dns_cache = false + istio = true + cloudrun = true + dns_cache = false node_pools = [ { @@ -336,8 +336,6 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | - - ## Requirements Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 68ae3ea714..a10536bd8a 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,6 @@ resource "google_container_cluster" "primary" { node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = "projects/${local.network_project_id}/global/networks/${var.network}" - dynamic "network_policy" { for_each = local.cluster_network_policy @@ -85,7 +84,6 @@ resource "google_container_cluster" "primary" { enable_components = var.monitoring_enabled_components } } - cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -107,13 +105,10 @@ resource "google_container_cluster" "primary" { } } } - vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling } - - default_max_pods_per_node = var.default_max_pods_per_node - + default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility @@ -162,7 +157,6 @@ resource "google_container_cluster" "primary" { horizontal_pod_autoscaling { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -234,7 +228,6 @@ resource "google_container_cluster" "primary" { end_time = maintenance_exclusion.value.end_time } } - } lifecycle { @@ -246,7 +239,6 @@ resource "google_container_cluster" "primary" { update = "45m" delete = "45m" } - node_pool { name = "default-pool" initial_node_count = var.initial_node_count @@ -352,7 +344,6 @@ resource "google_container_cluster" "primary" { security_group = authenticator_groups_config.value.security_group } } - notification_config { pubsub { enabled = var.notification_config_topic != "" ? true : false @@ -360,7 +351,6 @@ resource "google_container_cluster" "primary" { } } } - /****************************************** Create Container Cluster node pools *****************************************/ diff --git a/modules/beta-private-cluster-update-variant/dns.tf b/modules/beta-private-cluster-update-variant/dns.tf index 5dd9a8ee24..1a4c059a30 100644 --- a/modules/beta-private-cluster-update-variant/dns.tf +++ b/modules/beta-private-cluster-update-variant/dns.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster-update-variant/firewall.tf b/modules/beta-private-cluster-update-variant/firewall.tf index e6318ff45b..14c3e54cb0 100644 --- a/modules/beta-private-cluster-update-variant/firewall.tf +++ b/modules/beta-private-cluster-update-variant/firewall.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 476f434c93..37618874cd 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,6 @@ locals { master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - // Build a map of maps of node pools from a list of objects node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) @@ -81,7 +80,6 @@ locals { enabled = false provider = null }] - cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} diff --git a/modules/beta-private-cluster-update-variant/masq.tf b/modules/beta-private-cluster-update-variant/masq.tf index 2c45975992..b356aee251 100644 --- a/modules/beta-private-cluster-update-variant/masq.tf +++ b/modules/beta-private-cluster-update-variant/masq.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster-update-variant/networks.tf b/modules/beta-private-cluster-update-variant/networks.tf index 295263c29a..ace2ab0030 100644 --- a/modules/beta-private-cluster-update-variant/networks.tf +++ b/modules/beta-private-cluster-update-variant/networks.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index 87369882e6..27f413723e 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster-update-variant/sa.tf b/modules/beta-private-cluster-update-variant/sa.tf index b12b3befdb..eb63753625 100644 --- a/modules/beta-private-cluster-update-variant/sa.tf +++ b/modules/beta-private-cluster-update-variant/sa.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index e7284839f1..b0c7ceb9e8 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster-update-variant/variables_defaults.tf b/modules/beta-private-cluster-update-variant/variables_defaults.tf index 3cda3a15d8..ee5d60e6c7 100644 --- a/modules/beta-private-cluster-update-variant/variables_defaults.tf +++ b/modules/beta-private-cluster-update-variant/variables_defaults.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf index e68d00a3b3..4b405cdfa7 100644 --- a/modules/beta-private-cluster-update-variant/versions.tf +++ b/modules/beta-private-cluster-update-variant/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index adb88fca48..b28908fd4e 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -46,14 +46,14 @@ module "gke" { ip_range_pods = "us-central1-01-gke-01-pods" ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false - horizontal_pod_autoscaling = true network_policy = false + horizontal_pod_autoscaling = true enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" - istio = true - cloudrun = true - dns_cache = false + istio = true + cloudrun = true + dns_cache = false node_pools = [ { @@ -314,8 +314,6 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | - - ## Requirements Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index aff68b8e12..6bb5e8e459 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,6 @@ resource "google_container_cluster" "primary" { node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = "projects/${local.network_project_id}/global/networks/${var.network}" - dynamic "network_policy" { for_each = local.cluster_network_policy @@ -85,7 +84,6 @@ resource "google_container_cluster" "primary" { enable_components = var.monitoring_enabled_components } } - cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -107,13 +105,10 @@ resource "google_container_cluster" "primary" { } } } - vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling } - - default_max_pods_per_node = var.default_max_pods_per_node - + default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility @@ -162,7 +157,6 @@ resource "google_container_cluster" "primary" { horizontal_pod_autoscaling { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -234,7 +228,6 @@ resource "google_container_cluster" "primary" { end_time = maintenance_exclusion.value.end_time } } - } lifecycle { @@ -246,7 +239,6 @@ resource "google_container_cluster" "primary" { update = "45m" delete = "45m" } - node_pool { name = "default-pool" initial_node_count = var.initial_node_count @@ -352,7 +344,6 @@ resource "google_container_cluster" "primary" { security_group = authenticator_groups_config.value.security_group } } - notification_config { pubsub { enabled = var.notification_config_topic != "" ? true : false @@ -360,7 +351,6 @@ resource "google_container_cluster" "primary" { } } } - /****************************************** Create Container Cluster node pools *****************************************/ diff --git a/modules/beta-private-cluster/dns.tf b/modules/beta-private-cluster/dns.tf index 5dd9a8ee24..1a4c059a30 100644 --- a/modules/beta-private-cluster/dns.tf +++ b/modules/beta-private-cluster/dns.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster/firewall.tf b/modules/beta-private-cluster/firewall.tf index e6318ff45b..14c3e54cb0 100644 --- a/modules/beta-private-cluster/firewall.tf +++ b/modules/beta-private-cluster/firewall.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 476f434c93..37618874cd 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,6 @@ locals { master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - // Build a map of maps of node pools from a list of objects node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) @@ -81,7 +80,6 @@ locals { enabled = false provider = null }] - cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} diff --git a/modules/beta-private-cluster/masq.tf b/modules/beta-private-cluster/masq.tf index 2c45975992..b356aee251 100644 --- a/modules/beta-private-cluster/masq.tf +++ b/modules/beta-private-cluster/masq.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster/networks.tf b/modules/beta-private-cluster/networks.tf index 295263c29a..ace2ab0030 100644 --- a/modules/beta-private-cluster/networks.tf +++ b/modules/beta-private-cluster/networks.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index 87369882e6..27f413723e 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster/sa.tf b/modules/beta-private-cluster/sa.tf index b12b3befdb..eb63753625 100644 --- a/modules/beta-private-cluster/sa.tf +++ b/modules/beta-private-cluster/sa.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index e7284839f1..b0c7ceb9e8 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster/variables_defaults.tf b/modules/beta-private-cluster/variables_defaults.tf index 3cda3a15d8..ee5d60e6c7 100644 --- a/modules/beta-private-cluster/variables_defaults.tf +++ b/modules/beta-private-cluster/variables_defaults.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf index 0ecb84ea93..d18670b50a 100644 --- a/modules/beta-private-cluster/versions.tf +++ b/modules/beta-private-cluster/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 28d39f54f2..3b0fa1fc92 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -65,11 +65,11 @@ module "gke" { ip_range_pods = "us-central1-01-gke-01-pods" ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false - horizontal_pod_autoscaling = true network_policy = false - istio = true - cloudrun = true - dns_cache = false + horizontal_pod_autoscaling = true + istio = true + cloudrun = true + dns_cache = false node_pools = [ { @@ -323,8 +323,6 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | - - ## Requirements Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 9a67b1f798..fd75857c28 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,6 @@ resource "google_container_cluster" "primary" { node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = "projects/${local.network_project_id}/global/networks/${var.network}" - dynamic "network_policy" { for_each = local.cluster_network_policy @@ -85,7 +84,6 @@ resource "google_container_cluster" "primary" { enable_components = var.monitoring_enabled_components } } - cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -107,13 +105,10 @@ resource "google_container_cluster" "primary" { } } } - vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling } - - default_max_pods_per_node = var.default_max_pods_per_node - + default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility @@ -162,7 +157,6 @@ resource "google_container_cluster" "primary" { horizontal_pod_autoscaling { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -234,7 +228,6 @@ resource "google_container_cluster" "primary" { end_time = maintenance_exclusion.value.end_time } } - } lifecycle { @@ -246,7 +239,6 @@ resource "google_container_cluster" "primary" { update = "45m" delete = "45m" } - node_pool { name = "default-pool" initial_node_count = var.initial_node_count @@ -333,7 +325,6 @@ resource "google_container_cluster" "primary" { security_group = authenticator_groups_config.value.security_group } } - notification_config { pubsub { enabled = var.notification_config_topic != "" ? true : false @@ -341,7 +332,6 @@ resource "google_container_cluster" "primary" { } } } - /****************************************** Create Container Cluster node pools *****************************************/ diff --git a/modules/beta-public-cluster-update-variant/dns.tf b/modules/beta-public-cluster-update-variant/dns.tf index 5dd9a8ee24..1a4c059a30 100644 --- a/modules/beta-public-cluster-update-variant/dns.tf +++ b/modules/beta-public-cluster-update-variant/dns.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster-update-variant/firewall.tf b/modules/beta-public-cluster-update-variant/firewall.tf index b808dba1f3..6b71404b79 100644 --- a/modules/beta-public-cluster-update-variant/firewall.tf +++ b/modules/beta-public-cluster-update-variant/firewall.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index 94acbd8c3a..55af249c10 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,6 @@ locals { master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - // Build a map of maps of node pools from a list of objects node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) @@ -81,7 +80,6 @@ locals { enabled = false provider = null }] - cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} diff --git a/modules/beta-public-cluster-update-variant/masq.tf b/modules/beta-public-cluster-update-variant/masq.tf index 2c45975992..b356aee251 100644 --- a/modules/beta-public-cluster-update-variant/masq.tf +++ b/modules/beta-public-cluster-update-variant/masq.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster-update-variant/networks.tf b/modules/beta-public-cluster-update-variant/networks.tf index 295263c29a..ace2ab0030 100644 --- a/modules/beta-public-cluster-update-variant/networks.tf +++ b/modules/beta-public-cluster-update-variant/networks.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster-update-variant/outputs.tf b/modules/beta-public-cluster-update-variant/outputs.tf index 2cd7cb5c9e..289ccb0aea 100644 --- a/modules/beta-public-cluster-update-variant/outputs.tf +++ b/modules/beta-public-cluster-update-variant/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster-update-variant/sa.tf b/modules/beta-public-cluster-update-variant/sa.tf index b12b3befdb..eb63753625 100644 --- a/modules/beta-public-cluster-update-variant/sa.tf +++ b/modules/beta-public-cluster-update-variant/sa.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 94f737c3c1..564e019b2a 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster-update-variant/variables_defaults.tf b/modules/beta-public-cluster-update-variant/variables_defaults.tf index 3cda3a15d8..ee5d60e6c7 100644 --- a/modules/beta-public-cluster-update-variant/variables_defaults.tf +++ b/modules/beta-public-cluster-update-variant/variables_defaults.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster-update-variant/versions.tf b/modules/beta-public-cluster-update-variant/versions.tf index b0d16dd6e2..9fdf6935dd 100644 --- a/modules/beta-public-cluster-update-variant/versions.tf +++ b/modules/beta-public-cluster-update-variant/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 56b7db3377..cf329349f3 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -43,11 +43,11 @@ module "gke" { ip_range_pods = "us-central1-01-gke-01-pods" ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false - horizontal_pod_autoscaling = true network_policy = false - istio = true - cloudrun = true - dns_cache = false + horizontal_pod_autoscaling = true + istio = true + cloudrun = true + dns_cache = false node_pools = [ { @@ -301,8 +301,6 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | - - ## Requirements Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index ec1848a871..539739c9d6 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,6 @@ resource "google_container_cluster" "primary" { node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = "projects/${local.network_project_id}/global/networks/${var.network}" - dynamic "network_policy" { for_each = local.cluster_network_policy @@ -85,7 +84,6 @@ resource "google_container_cluster" "primary" { enable_components = var.monitoring_enabled_components } } - cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -107,13 +105,10 @@ resource "google_container_cluster" "primary" { } } } - vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling } - - default_max_pods_per_node = var.default_max_pods_per_node - + default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility @@ -162,7 +157,6 @@ resource "google_container_cluster" "primary" { horizontal_pod_autoscaling { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -234,7 +228,6 @@ resource "google_container_cluster" "primary" { end_time = maintenance_exclusion.value.end_time } } - } lifecycle { @@ -246,7 +239,6 @@ resource "google_container_cluster" "primary" { update = "45m" delete = "45m" } - node_pool { name = "default-pool" initial_node_count = var.initial_node_count @@ -333,7 +325,6 @@ resource "google_container_cluster" "primary" { security_group = authenticator_groups_config.value.security_group } } - notification_config { pubsub { enabled = var.notification_config_topic != "" ? true : false @@ -341,7 +332,6 @@ resource "google_container_cluster" "primary" { } } } - /****************************************** Create Container Cluster node pools *****************************************/ diff --git a/modules/beta-public-cluster/dns.tf b/modules/beta-public-cluster/dns.tf index 5dd9a8ee24..1a4c059a30 100644 --- a/modules/beta-public-cluster/dns.tf +++ b/modules/beta-public-cluster/dns.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster/firewall.tf b/modules/beta-public-cluster/firewall.tf index b808dba1f3..6b71404b79 100644 --- a/modules/beta-public-cluster/firewall.tf +++ b/modules/beta-public-cluster/firewall.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 94acbd8c3a..55af249c10 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,6 @@ locals { master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - // Build a map of maps of node pools from a list of objects node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) @@ -81,7 +80,6 @@ locals { enabled = false provider = null }] - cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} diff --git a/modules/beta-public-cluster/masq.tf b/modules/beta-public-cluster/masq.tf index 2c45975992..b356aee251 100644 --- a/modules/beta-public-cluster/masq.tf +++ b/modules/beta-public-cluster/masq.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster/networks.tf b/modules/beta-public-cluster/networks.tf index 295263c29a..ace2ab0030 100644 --- a/modules/beta-public-cluster/networks.tf +++ b/modules/beta-public-cluster/networks.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index 2cd7cb5c9e..289ccb0aea 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster/sa.tf b/modules/beta-public-cluster/sa.tf index b12b3befdb..eb63753625 100644 --- a/modules/beta-public-cluster/sa.tf +++ b/modules/beta-public-cluster/sa.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 94f737c3c1..564e019b2a 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster/variables_defaults.tf b/modules/beta-public-cluster/variables_defaults.tf index 3cda3a15d8..ee5d60e6c7 100644 --- a/modules/beta-public-cluster/variables_defaults.tf +++ b/modules/beta-public-cluster/variables_defaults.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf index 43ba811e69..e34e3ac3c0 100644 --- a/modules/beta-public-cluster/versions.tf +++ b/modules/beta-public-cluster/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index f586505f5d..993f2be40b 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -68,8 +68,8 @@ module "gke" { ip_range_pods = "us-central1-01-gke-01-pods" ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false - horizontal_pod_autoscaling = true network_policy = false + horizontal_pod_autoscaling = true enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" @@ -287,8 +287,6 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | - - ## Requirements Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index f1b80fd7e4..4a79dbe6c2 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,6 @@ resource "google_container_cluster" "primary" { node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = "projects/${local.network_project_id}/global/networks/${var.network}" - dynamic "network_policy" { for_each = local.cluster_network_policy @@ -55,7 +54,6 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service - cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -75,13 +73,10 @@ resource "google_container_cluster" "primary" { } } } - vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling } - - default_max_pods_per_node = var.default_max_pods_per_node - + default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization dynamic "master_authorized_networks_config" { @@ -111,7 +106,6 @@ resource "google_container_cluster" "primary" { horizontal_pod_autoscaling { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -139,7 +133,6 @@ resource "google_container_cluster" "primary" { update = "45m" delete = "45m" } - node_pool { name = "default-pool" initial_node_count = var.initial_node_count @@ -231,9 +224,7 @@ resource "google_container_cluster" "primary" { security_group = authenticator_groups_config.value.security_group } } - } - /****************************************** Create Container Cluster node pools *****************************************/ diff --git a/modules/private-cluster-update-variant/dns.tf b/modules/private-cluster-update-variant/dns.tf index 5dd9a8ee24..1a4c059a30 100644 --- a/modules/private-cluster-update-variant/dns.tf +++ b/modules/private-cluster-update-variant/dns.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster-update-variant/firewall.tf b/modules/private-cluster-update-variant/firewall.tf index d913356276..c9b4d29972 100644 --- a/modules/private-cluster-update-variant/firewall.tf +++ b/modules/private-cluster-update-variant/firewall.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 686bc61fa2..5702f92d81 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,6 @@ locals { master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - // Build a map of maps of node pools from a list of objects node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) @@ -81,7 +80,6 @@ locals { provider = null }] - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group }] diff --git a/modules/private-cluster-update-variant/masq.tf b/modules/private-cluster-update-variant/masq.tf index 2c45975992..b356aee251 100644 --- a/modules/private-cluster-update-variant/masq.tf +++ b/modules/private-cluster-update-variant/masq.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster-update-variant/networks.tf b/modules/private-cluster-update-variant/networks.tf index 295263c29a..ace2ab0030 100644 --- a/modules/private-cluster-update-variant/networks.tf +++ b/modules/private-cluster-update-variant/networks.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index 98336bf5bd..62f205cda7 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster-update-variant/sa.tf b/modules/private-cluster-update-variant/sa.tf index b12b3befdb..eb63753625 100644 --- a/modules/private-cluster-update-variant/sa.tf +++ b/modules/private-cluster-update-variant/sa.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 7f6cce8e2c..0bd3341d92 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster-update-variant/variables_defaults.tf b/modules/private-cluster-update-variant/variables_defaults.tf index 70ac8ba1c1..e7f52e3d4b 100644 --- a/modules/private-cluster-update-variant/variables_defaults.tf +++ b/modules/private-cluster-update-variant/variables_defaults.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf index 075d0700be..7ae6bea1e8 100644 --- a/modules/private-cluster-update-variant/versions.tf +++ b/modules/private-cluster-update-variant/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 822e58139c..0bb0e7a400 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -46,8 +46,8 @@ module "gke" { ip_range_pods = "us-central1-01-gke-01-pods" ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false - horizontal_pod_autoscaling = true network_policy = false + horizontal_pod_autoscaling = true enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" @@ -265,8 +265,6 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | - - ## Requirements Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index e700101801..74ad14a293 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,6 @@ resource "google_container_cluster" "primary" { node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = "projects/${local.network_project_id}/global/networks/${var.network}" - dynamic "network_policy" { for_each = local.cluster_network_policy @@ -55,7 +54,6 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service - cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -75,13 +73,10 @@ resource "google_container_cluster" "primary" { } } } - vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling } - - default_max_pods_per_node = var.default_max_pods_per_node - + default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization dynamic "master_authorized_networks_config" { @@ -111,7 +106,6 @@ resource "google_container_cluster" "primary" { horizontal_pod_autoscaling { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -139,7 +133,6 @@ resource "google_container_cluster" "primary" { update = "45m" delete = "45m" } - node_pool { name = "default-pool" initial_node_count = var.initial_node_count @@ -231,9 +224,7 @@ resource "google_container_cluster" "primary" { security_group = authenticator_groups_config.value.security_group } } - } - /****************************************** Create Container Cluster node pools *****************************************/ diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf index 5dd9a8ee24..1a4c059a30 100644 --- a/modules/private-cluster/dns.tf +++ b/modules/private-cluster/dns.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster/firewall.tf b/modules/private-cluster/firewall.tf index d913356276..c9b4d29972 100644 --- a/modules/private-cluster/firewall.tf +++ b/modules/private-cluster/firewall.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 686bc61fa2..5702f92d81 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,6 @@ locals { master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - // Build a map of maps of node pools from a list of objects node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) @@ -81,7 +80,6 @@ locals { provider = null }] - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group }] diff --git a/modules/private-cluster/masq.tf b/modules/private-cluster/masq.tf index 2c45975992..b356aee251 100644 --- a/modules/private-cluster/masq.tf +++ b/modules/private-cluster/masq.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster/networks.tf b/modules/private-cluster/networks.tf index 295263c29a..ace2ab0030 100644 --- a/modules/private-cluster/networks.tf +++ b/modules/private-cluster/networks.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 98336bf5bd..62f205cda7 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster/sa.tf b/modules/private-cluster/sa.tf index b12b3befdb..eb63753625 100644 --- a/modules/private-cluster/sa.tf +++ b/modules/private-cluster/sa.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 7f6cce8e2c..0bd3341d92 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster/variables_defaults.tf b/modules/private-cluster/variables_defaults.tf index 70ac8ba1c1..e7f52e3d4b 100644 --- a/modules/private-cluster/variables_defaults.tf +++ b/modules/private-cluster/variables_defaults.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf index ff625a598a..245bc86668 100644 --- a/modules/private-cluster/versions.tf +++ b/modules/private-cluster/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/safer-cluster-update-variant/main.tf b/modules/safer-cluster-update-variant/main.tf index 6242fc4665..490c47b0c8 100644 --- a/modules/safer-cluster-update-variant/main.tf +++ b/modules/safer-cluster-update-variant/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/safer-cluster-update-variant/outputs.tf b/modules/safer-cluster-update-variant/outputs.tf index 9846251c16..5be616ff66 100644 --- a/modules/safer-cluster-update-variant/outputs.tf +++ b/modules/safer-cluster-update-variant/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/safer-cluster-update-variant/variables.tf b/modules/safer-cluster-update-variant/variables.tf index c8cc2f8a0b..ef38f36bf5 100644 --- a/modules/safer-cluster-update-variant/variables.tf +++ b/modules/safer-cluster-update-variant/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/safer-cluster-update-variant/versions.tf b/modules/safer-cluster-update-variant/versions.tf index c49883a9a8..e9e556a245 100644 --- a/modules/safer-cluster-update-variant/versions.tf +++ b/modules/safer-cluster-update-variant/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 567df7c61e..d372600a71 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/safer-cluster/outputs.tf b/modules/safer-cluster/outputs.tf index 9846251c16..5be616ff66 100644 --- a/modules/safer-cluster/outputs.tf +++ b/modules/safer-cluster/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index c8cc2f8a0b..ef38f36bf5 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf index 6f695fe8bb..cf2dc5fe46 100644 --- a/modules/safer-cluster/versions.tf +++ b/modules/safer-cluster/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/networks.tf b/networks.tf index 295263c29a..ace2ab0030 100644 --- a/networks.tf +++ b/networks.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/outputs.tf b/outputs.tf index a6f20a6f3b..c28719639a 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sa.tf b/sa.tf index b12b3befdb..eb63753625 100644 --- a/sa.tf +++ b/sa.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/fixtures/simple_autopilot_private/example.tf b/test/fixtures/simple_autopilot_private/example.tf new file mode 100644 index 0000000000..b50af16964 --- /dev/null +++ b/test/fixtures/simple_autopilot_private/example.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/simple_autopilot_private" + + project_id = var.project_ids[0] + region = var.region +} diff --git a/test/fixtures/simple_autopilot_private/outputs.tf b/test/fixtures/simple_autopilot_private/outputs.tf new file mode 100644 index 0000000000..1ff0731647 --- /dev/null +++ b/test/fixtures/simple_autopilot_private/outputs.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = module.example.project_id +} + +output "region" { + value = module.example.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.example.cluster_name +} +output "location" { + value = module.example.location +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.example.master_kubernetes_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.example.kubernetes_endpoint +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.example.ca_certificate + sensitive = true +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.example.service_account +} diff --git a/test/fixtures/simple_autopilot_private/variables.tf b/test/fixtures/simple_autopilot_private/variables.tf new file mode 100644 index 0000000000..4a7c600e6a --- /dev/null +++ b/test/fixtures/simple_autopilot_private/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" +} + +variable "region" { + description = "The GCP region to create and test resources in" + default = "us-central1" +} diff --git a/test/fixtures/simple_autopilot_public/example.tf b/test/fixtures/simple_autopilot_public/example.tf new file mode 100644 index 0000000000..c58ad440ec --- /dev/null +++ b/test/fixtures/simple_autopilot_public/example.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/simple_autopilot_public" + + project_id = var.project_ids[0] + region = var.region +} diff --git a/test/fixtures/simple_autopilot_public/outputs.tf b/test/fixtures/simple_autopilot_public/outputs.tf new file mode 100644 index 0000000000..1ff0731647 --- /dev/null +++ b/test/fixtures/simple_autopilot_public/outputs.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = module.example.project_id +} + +output "region" { + value = module.example.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.example.cluster_name +} +output "location" { + value = module.example.location +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.example.master_kubernetes_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.example.kubernetes_endpoint +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.example.ca_certificate + sensitive = true +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.example.service_account +} diff --git a/test/fixtures/simple_autopilot_public/variables.tf b/test/fixtures/simple_autopilot_public/variables.tf new file mode 100644 index 0000000000..4a7c600e6a --- /dev/null +++ b/test/fixtures/simple_autopilot_public/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" +} + +variable "region" { + description = "The GCP region to create and test resources in" + default = "us-central1" +} diff --git a/test/integration/simple_autopilot_private/controls/gcloud.rb b/test/integration/simple_autopilot_private/controls/gcloud.rb new file mode 100644 index 0000000000..6ad7afecd9 --- /dev/null +++ b/test/integration/simple_autopilot_private/controls/gcloud.rb @@ -0,0 +1,68 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is autopilot" do + expect(data['autopilot']['enabled']).to eq true + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses the private endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to include( + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + ) + end + end + end +end diff --git a/test/integration/simple_autopilot_private/inspec.yml b/test/integration/simple_autopilot_private/inspec.yml new file mode 100644 index 0000000000..b477741e83 --- /dev/null +++ b/test/integration/simple_autopilot_private/inspec.yml @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: simple_regional +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string diff --git a/test/integration/simple_autopilot_public/controls/gcloud.rb b/test/integration/simple_autopilot_public/controls/gcloud.rb new file mode 100644 index 0000000000..7c2d5f01d6 --- /dev/null +++ b/test/integration/simple_autopilot_public/controls/gcloud.rb @@ -0,0 +1,64 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is autopilot" do + expect(data['autopilot']['enabled']).to eq true + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses public nodes and master endpoint" do + expect(data['privateClusterConfig']).to eq nil + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to include( + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + ) + end + end + end +end diff --git a/test/integration/simple_autopilot_public/inspec.yml b/test/integration/simple_autopilot_public/inspec.yml new file mode 100644 index 0000000000..b477741e83 --- /dev/null +++ b/test/integration/simple_autopilot_public/inspec.yml @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: simple_regional +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string diff --git a/test/setup/main.tf b/test/setup/main.tf index a4bee2d789..e3015cbda8 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -75,10 +75,8 @@ module "gke-project-2" { # apis as documented https://cloud.google.com/service-mesh/docs/scripted-install/reference#setting_up_your_project module "gke-project-asm" { - source = "github.com/terraform-google-modules/terraform-google-project-factory.git?ref=master" - - #source = "terraform-google-modules/project-factory/google" - #version = "~> 11.3" + source = "terraform-google-modules/project-factory/google" + version = "~> 11.3" name = "ci-gke-asm-${random_id.random_project_id_suffix.hex}" random_project_id = true diff --git a/variables.tf b/variables.tf index 9fa190202a..e01a012cbe 100644 --- a/variables.tf +++ b/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/variables_defaults.tf b/variables_defaults.tf index 70ac8ba1c1..e7f52e3d4b 100644 --- a/variables_defaults.tf +++ b/variables_defaults.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/versions.tf b/versions.tf index 5508b2130d..990c289615 100644 --- a/versions.tf +++ b/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License.