From a0f5f7b9b48bc9161a18b148c0d59b5e441f23b2 Mon Sep 17 00:00:00 2001 From: pp Date: Fri, 18 Oct 2019 14:42:34 +0300 Subject: [PATCH 1/3] Moved sanbox_config to node_pool resource (Fix #240) --- autogen/cluster.tf | 16 ++++++++-------- .../cluster.tf | 16 ++++++++-------- modules/beta-private-cluster/cluster.tf | 16 ++++++++-------- modules/beta-public-cluster/cluster.tf | 16 ++++++++-------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/autogen/cluster.tf b/autogen/cluster.tf index 296b2818df..035eeb0cf7 100644 --- a/autogen/cluster.tf +++ b/autogen/cluster.tf @@ -167,14 +167,6 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } - - dynamic "sandbox_config" { - for_each = local.cluster_sandbox_enabled - - content { - sandbox_type = sandbox_config.value - } - } {% endif %} } } @@ -415,6 +407,14 @@ resource "google_container_node_pool" "pools" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } {% endif %} } diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index cf1def945d..2348150d19 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -158,14 +158,6 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } - - dynamic "sandbox_config" { - for_each = local.cluster_sandbox_enabled - - content { - sandbox_type = sandbox_config.value - } - } } } @@ -386,6 +378,14 @@ resource "google_container_node_pool" "pools" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } } lifecycle { diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index c481c69a35..56f40ed17a 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -158,14 +158,6 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } - - dynamic "sandbox_config" { - for_each = local.cluster_sandbox_enabled - - content { - sandbox_type = sandbox_config.value - } - } } } @@ -314,6 +306,14 @@ resource "google_container_node_pool" "pools" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } } lifecycle { diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index a264e932b9..e37b2b3b31 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -158,14 +158,6 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } - - dynamic "sandbox_config" { - for_each = local.cluster_sandbox_enabled - - content { - sandbox_type = sandbox_config.value - } - } } } @@ -309,6 +301,14 @@ resource "google_container_node_pool" "pools" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } } lifecycle { From e90d20bac2a364ebdaec067273ec2c676790a977 Mon Sep 17 00:00:00 2001 From: pp Date: Mon, 21 Oct 2019 11:05:51 +0300 Subject: [PATCH 2/3] Created test for sandbox (Fix #252) --- .kitchen.yml | 7 ++ examples/simple_regional_beta/README.md | 11 +- examples/simple_regional_beta/main.tf | 35 +++--- examples/simple_regional_beta/test_outputs.tf | 4 - examples/simple_regional_beta/variables.tf | 33 +++++- test/fixtures/sandbox_enabled/example.tf | 40 +++++++ test/fixtures/sandbox_enabled/network.tf | 48 +++++++++ test/fixtures/sandbox_enabled/outputs.tf | 1 + test/fixtures/sandbox_enabled/variables.tf | 1 + .../sandbox_enabled/controls/gcloud.rb | 102 ++++++++++++++++++ test/integration/sandbox_enabled/inspec.yml | 17 +++ 11 files changed, 270 insertions(+), 29 deletions(-) create mode 100644 test/fixtures/sandbox_enabled/example.tf create mode 100644 test/fixtures/sandbox_enabled/network.tf create mode 120000 test/fixtures/sandbox_enabled/outputs.tf create mode 120000 test/fixtures/sandbox_enabled/variables.tf create mode 100644 test/integration/sandbox_enabled/controls/gcloud.rb create mode 100644 test/integration/sandbox_enabled/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 9f5df5a03e..61b8aae029 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -131,3 +131,10 @@ suites: systems: - name: workload_metadata_config backend: local + - name: "sandbox_enabled" + driver: + root_module_directory: test/fixtures/sandbox_enabled + verifier: + systems: + - name: sandbox_enabled + backend: local diff --git a/examples/simple_regional_beta/README.md b/examples/simple_regional_beta/README.md index bd676115b9..02d0dba224 100644 --- a/examples/simple_regional_beta/README.md +++ b/examples/simple_regional_beta/README.md @@ -2,8 +2,7 @@ This example illustrates how to create a simple cluster with beta features. -[^]: (autogen_docs_start) - + ## Inputs | Name | Description | Type | Default | Required | @@ -11,13 +10,16 @@ This example illustrates how to create a simple cluster with beta features. | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | | istio | Boolean to enable / disable Istio | string | `"true"` | no | | network | The VPC network to host the cluster in | string | n/a | yes | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | subnetwork | The subnetwork to host the cluster in | string | n/a | yes | ## Outputs @@ -27,7 +29,6 @@ This example illustrates how to create a simple cluster with beta features. | ca\_certificate | | | client\_token | | | cluster\_name | Cluster name | -| credentials\_path | | | ip\_range\_pods | The secondary IP range used for pods | | ip\_range\_services | The secondary IP range used for services | | kubernetes\_endpoint | | @@ -40,7 +41,7 @@ This example illustrates how to create a simple cluster with beta features. | subnetwork | | | zones | List of zones in which the cluster resides | -[^]: (autogen_docs_end) + To provision this example, run the following from within this directory: - `terraform init` to get the plugins diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf index fc95090ede..b75fdaa613 100644 --- a/examples/simple_regional_beta/main.tf +++ b/examples/simple_regional_beta/main.tf @@ -19,25 +19,28 @@ locals { } provider "google-beta" { - version = "~> 2.12.0" - credentials = file(var.credentials_path) - region = var.region + version = "~> 2.12.0" + region = var.region } module "gke" { - source = "../../modules/beta-public-cluster/" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = true - region = var.region - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - create_service_account = false - service_account = var.compute_engine_service_account - istio = var.istio - cloudrun = var.cloudrun + source = "../../modules/beta-public-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = false + service_account = var.compute_engine_service_account + istio = var.istio + cloudrun = var.cloudrun + node_metadata = var.node_metadata + sandbox_enabled = var.sandbox_enabled + remove_default_node_pool = var.remove_default_node_pool + node_pools = var.node_pools } data "google_client_config" "default" { diff --git a/examples/simple_regional_beta/test_outputs.tf b/examples/simple_regional_beta/test_outputs.tf index f250fef192..e64c40e477 100644 --- a/examples/simple_regional_beta/test_outputs.tf +++ b/examples/simple_regional_beta/test_outputs.tf @@ -21,10 +21,6 @@ output "project_id" { value = var.project_id } -output "credentials_path" { - value = var.credentials_path -} - output "region" { value = module.gke.region } diff --git a/examples/simple_regional_beta/variables.tf b/examples/simple_regional_beta/variables.tf index 1da408a790..ed16642774 100644 --- a/examples/simple_regional_beta/variables.tf +++ b/examples/simple_regional_beta/variables.tf @@ -18,10 +18,6 @@ variable "project_id" { description = "The project ID to host the cluster in" } -variable "credentials_path" { - description = "The path to the GCP credentials JSON file" -} - variable "cluster_name_suffix" { description = "A suffix to append to the default cluster name" default = "" @@ -60,3 +56,32 @@ variable "cloudrun" { description = "Boolean to enable / disable CloudRun" default = true } + +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "SECURE" + type = string +} + +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + +variable "remove_default_node_pool" { + type = bool + description = "Remove default node pool while setting up the cluster" + default = false +} + +variable "node_pools" { + type = list(map(string)) + description = "List of maps containing node pools" + + default = [ + { + name = "default-node-pool" + }, + ] +} diff --git a/test/fixtures/sandbox_enabled/example.tf b/test/fixtures/sandbox_enabled/example.tf new file mode 100644 index 0000000000..05b7edfd9e --- /dev/null +++ b/test/fixtures/sandbox_enabled/example.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/simple_regional_beta" + + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = var.compute_engine_service_account + istio = false + cloudrun = false + node_metadata = "UNSPECIFIED" + sandbox_enabled = true + remove_default_node_pool = true + + node_pools = [ + { + name = "default-node-pool" + image_type = "COS_CONTAINERD" + }, + ] +} diff --git a/test/fixtures/sandbox_enabled/network.tf b/test/fixtures/sandbox_enabled/network.tf new file mode 100644 index 0000000000..5d34d43748 --- /dev/null +++ b/test/fixtures/sandbox_enabled/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_id +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/sandbox_enabled/outputs.tf b/test/fixtures/sandbox_enabled/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/sandbox_enabled/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/sandbox_enabled/variables.tf b/test/fixtures/sandbox_enabled/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/sandbox_enabled/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/sandbox_enabled/controls/gcloud.rb b/test/integration/sandbox_enabled/controls/gcloud.rb new file mode 100644 index 0000000000..eb0ffdaf46 --- /dev/null +++ b/test/integration/sandbox_enabled/controls/gcloud.rb @@ -0,0 +1,102 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses public nodes and master endpoint" do + expect(data['privateClusterConfig']).to eq nil + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + }) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "is the expected image type" do + expect(node_pools).to include( + including( + "config" => including( + "imageType" => "COS_CONTAINERD", + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + "sandbox.gke.io/runtime" => "gvisor", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + end + end +end diff --git a/test/integration/sandbox_enabled/inspec.yml b/test/integration/sandbox_enabled/inspec.yml new file mode 100644 index 0000000000..0454937a36 --- /dev/null +++ b/test/integration/sandbox_enabled/inspec.yml @@ -0,0 +1,17 @@ +name: sandbox_enabled +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string From 8d8451c7112f483a2fc40424f48820688d7c289b Mon Sep 17 00:00:00 2001 From: pp Date: Fri, 25 Oct 2019 11:29:49 +0300 Subject: [PATCH 3/3] Added sandbox-enabled-local to int tests on CI * Fixed lint --- build/int.cloudbuild.yaml | 20 ++++++++++++++++++++ examples/simple_regional_beta/README.md | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 379205db45..a4ce07ab4a 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -221,6 +221,26 @@ steps: - verify workload-metadata-config-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-metadata-config-local'] +- id: create sandbox-enabled-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create sandbox-enabled-local'] +- id: converge sandbox-enabled-local + waitFor: + - create sandbox-enabled-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge sandbox-enabled-local'] +- id: verify sandbox-enabled-local + waitFor: + - converge sandbox-enabled-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify sandbox-enabled-local'] +- id: destroy sandbox-enabled-local + waitFor: + - verify sandbox-enabled-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy sandbox-enabled-local'] tags: - 'ci' - 'integration' diff --git a/examples/simple_regional_beta/README.md b/examples/simple_regional_beta/README.md index 02d0dba224..72bb221d9f 100644 --- a/examples/simple_regional_beta/README.md +++ b/examples/simple_regional_beta/README.md @@ -37,7 +37,7 @@ This example illustrates how to create a simple cluster with beta features. | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides |