Build Network Fixtures and Run Tests with Kitchen-Terraform

.gitignore

@@ -38,12 +38,9 @@ crash.log
 **/.kitchen.local.yml
 **/Gemfile.lock
 
-# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
-# .tfvars files are managed as part of configuration and so should be included in
-# version control.
-#
-# example.tfvars
-**/terraform.tfvars
+test/fixtures/shared/terraform.tfvars
 
 test/integration/gcloud/config.sh
 test/integration/tmp
+
+credentials.json

.kitchen.yml

@@ -0,0 +1,98 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+driver:
+  name: "terraform"
+  command_timeout: 1800
+
+provisioner:
+  name: "terraform"
+
+platforms:
+  - name: local
+
+suites:
+  - name: "deploy_service"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: test/fixtures/deploy_service
+    verifier:
+      name: terraform
+      systems:
+        - name: deploy_service
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "node_pool"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: test/fixtures/node_pool
+    verifier:
+      name: terraform
+      systems:
+        - name: node_pool
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "shared_vpc"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: test/fixtures/shared_vpc
+    verifier:
+      name: terraform
+      systems:
+        - name: shared_vpc
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "simple_regional"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: test/fixtures/simple_regional
+    verifier:
+      name: terraform
+      systems:
+        - name: simple_regional
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "simple_zonal"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: test/fixtures/simple_zonal
+    verifier:
+      name: terraform
+      systems:
+        - name: simple_zonal
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "stub_domains"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: test/fixtures/stub_domains
+    verifier:
+      name: terraform
+      systems:
+        - name: stub_domains
+          backend: local
+    provisioner:
+      name: terraform

.ruby-version

@@ -0,0 +1 @@
+2.5.3

test/integration/gcloud/Gemfile → Gemfile

@@ -12,11 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ruby '2.4.2'
+ruby "~> 2.5"
 
 source 'https://rubygems.org/' do
-  gem 'googleauth'
-  gem 'google-api-client'
-  gem 'kitchen-terraform', '~> 3.3'
-  gem 'kitchen-inspec', :git => 'https://github.com/inspec/kitchen-inspec.git', :ref => '0590f1b'
+  gem "kitchen-terraform", "~> 4.0"
+  gem "kubeclient", "~> 4.0"
+  gem "rest-client", "~> 2.0"
 end

Makefile

@@ -15,6 +15,17 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
+# Docker build config variables
+BUILD_TERRAFORM_VERSION ?= 0.11.10
+BUILD_CLOUD_SDK_VERSION ?= 216.0.0
+BUILD_PROVIDER_GOOGLE_VERSION ?= 1.17.1
+BUILD_PROVIDER_GSUITE_VERSION ?= 0.1.8
+DOCKER_IMAGE_TERRAFORM := cftk/terraform
+DOCKER_TAG_TERRAFORM ?= ${BUILD_TERRAFORM_VERSION}_${BUILD_CLOUD_SDK_VERSION}_${BUILD_PROVIDER_GOOGLE_VERSION}_${BUILD_PROVIDER_GSUITE_VERSION}
+BUILD_RUBY_VERSION := 2.5.3
+DOCKER_IMAGE_KITCHEN_TERRAFORM := cftk/kitchen_terraform
+DOCKER_TAG_KITCHEN_TERRAFORM ?= ${BUILD_TERRAFORM_VERSION}_${BUILD_CLOUD_SDK_VERSION}_${BUILD_PROVIDER_GOOGLE_VERSION}_${BUILD_PROVIDER_GSUITE_VERSION}
+
 # All is the first target in the file so it will get picked up when you just run 'make' on its own
 all: check_shell check_python check_golang check_terraform check_docker check_base_files test_check_headers check_headers check_trailing_whitespace generate_docs
 
@@ -63,20 +74,80 @@ check_headers:
 	@echo "Checking file headers"
 	@python test/verify_boilerplate.py
 
+# Integration tests
+.PHONY: test_integration
+test_integration:
+	bundle install
+	bundle exec kitchen create
+	bundle exec kitchen converge
+	bundle exec kitchen converge
+	bundle exec kitchen verify
+	bundle exec kitchen destroy
+
 .PHONY: generate_docs
 generate_docs:
 	@source test/make.sh && generate_docs
 
-# Integration tests
-
-.PHONY: regional_test_integration
-regional_test_integration:
-	./test/integration/gcloud/run.sh regional
-
-.PHONY: zonal_test_integration
-zonal_test_integration:
-	./test/integration/gcloud/run.sh zonal
-
-.PHONY: test_integration
-test_integration: regional_test_integration zonal_test_integration
-	@echo "Running tests for regional and zonal clusters"
+# Versioning
+.PHONY: version
+version:
+	@source helpers/version-repo.sh
+
+# Build Docker
+.PHONY: docker_build_terraform
+docker_build_terraform:
+	docker build -f build/docker/terraform/Dockerfile \
+		--build-arg BUILD_TERRAFORM_VERSION=${BUILD_TERRAFORM_VERSION} \
+		--build-arg BUILD_CLOUD_SDK_VERSION=${BUILD_CLOUD_SDK_VERSION} \
+		--build-arg BUILD_PROVIDER_GOOGLE_VERSION=${BUILD_PROVIDER_GOOGLE_VERSION} \
+		--build-arg BUILD_PROVIDER_GSUITE_VERSION=${BUILD_PROVIDER_GSUITE_VERSION} \
+		--build-arg CREDENTIALS_FILE=${CREDENTIALS_FILE} \
+		-t ${DOCKER_IMAGE_TERRAFORM}:${DOCKER_TAG_TERRAFORM} .
+
+.PHONY: docker_build_kitchen_terraform
+docker_build_kitchen_terraform:
+	docker build -f build/docker/kitchen_terraform/Dockerfile \
+		--build-arg BUILD_TERRAFORM_IMAGE="${DOCKER_IMAGE_TERRAFORM}:${DOCKER_TAG_TERRAFORM}" \
+		--build-arg BUILD_RUBY_VERSION="${BUILD_RUBY_VERSION}" \
+		--build-arg CREDENTIALS_FILE="${CREDENTIALS_FILE}" \
+		-t ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} .
+
+# Run docker
+.PHONY: docker_run
+docker_run:
+	docker run --rm -it \
+		-v $(CURDIR):/cftk/workdir \
+		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		/bin/bash
+
+.PHONY: docker_create
+docker_create: docker_build_terraform docker_build_kitchen_terraform
+	docker run --rm -it \
+		-v $(CURDIR):/cftk/workdir \
+		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		/bin/bash -c "kitchen create"
+
+.PHONY: docker_converge
+docker_converge:
+	docker run --rm -it \
+		-v $(CURDIR):/cftk/workdir \
+		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		/bin/bash -c "kitchen converge && kitchen converge"
+
+.PHONY: docker_verify
+docker_verify:
+	docker run --rm -it \
+		-v $(CURDIR):/cftk/workdir \
+		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		/bin/bash -c "kitchen verify"
+
+.PHONY: docker_destroy
+docker_destroy:
+	docker run --rm -it \
+		-v $(CURDIR):/cftk/workdir \
+		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		/bin/bash -c "kitchen destroy"
+
+.PHONY: test_integration_docker
+test_integration_docker: docker_create docker_converge docker_verify docker_destroy
+	@echo "Running test-kitchen tests in docker"

README.md

@@ -97,7 +97,7 @@ Then perform the following commands on the root folder:
 | ip_range_pods | The secondary ip range to use for pods | string | - | yes |
 | ip_range_services | The secondary ip range to use for pods | string | - | yes |
 | kubernetes_dashboard | Enable kubernetes dashboard addon | string | `false` | no |
-| kubernetes_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `1.10.6-gke.2` | no |
+| kubernetes_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `latest` | no |
 | logging_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `logging.googleapis.com` | no |
 | maintenance_start_time | Time window specified for daily maintenance operations in RFC3339 format | string | `05:00` | no |
 | master_authorized_networks_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)<br><br>  ### example format ###   master_authorized_networks_config = [{     cidr_blocks = [{       cidr_block   = "10.0.0.0/8"       display_name = "example_network"     }],   }] | list | `<list>` | no |
@@ -115,6 +115,7 @@ Then perform the following commands on the root folder:
 | project_id | The project ID to host the cluster in (required) | string | - | yes |
 | region | The region to host the cluster in (required) | string | - | yes |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `true` | no |
+| service_account | The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account | string | `` | no |
 | stub_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | - | yes |
 | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `<list>` | no |
@@ -163,12 +164,12 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) plugin v1.8.0
 
 ### Configure a Service Account
-In order to execute this module you must have a Service Account with the following:
-
-#### IAM Roles
-The service account with the following roles:
-- roles/compute.viewer on the project
-- roles/container.clusterAdmin on the project
+In order to execute this module you must have a Service Account with the
+following project roles:
+- roles/compute.viewer
+- roles/container.clusterAdmin
+- roles/container.developer
+- roles/iam.serviceAccountUser
 
 ### Enable APIs
 In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created:
@@ -199,7 +200,6 @@ The project has the following folders and files:
 ### Requirements
 - [bundler](https://github.com/bundler/bundler)
 - [gcloud](https://cloud.google.com/sdk/install)
-- [jq](https://stedolan.github.io/jq/) 1.5
 - [terraform-docs](https://github.com/segmentio/terraform-docs/releases) 0.3.0
 
 ### Autogeneration of documentation from .tf files
@@ -209,27 +209,59 @@ make generate_docs
 ```
 
 ### Integration test
-#### Terraform integration tests
-The integration tests for this module leverage [kitchen-terraform](https://github.com/newcontext-oss/kitchen-terraform) and [kitchen-inspec](https://github.com/inspec/kitchen-inspec).
-
-The tests will do the following:
-- Perform `bundle install` command
-  - Installs `kitchen-terraform` and `kitchen-inspec` gems
-- Perform `kitchen create` command
-  - Performs a `terraform init`
-- Perform `kitchen converge` command
-  - Performs a `terraform apply -auto-approve`
-- Perform `kitchen validate` command
-  - Performs inspec tests.
-    - Shell out to `gcloud` to validate expected resources in GCP.
-    - Shell out to `kubectl` to validate expected resource in Kubernetes.
-    - Shell out to `terraform` to validate outputs.
-- Permos `kitchen destroy` command
-  - Performs a `terraform destroy -force`
-
-You can use the following command to run the integration test in the root folder
-
-  `make test_integration`
+
+Integration tests are run though [test-kitchen](https://github.com/test-kitchen/test-kitchen), [kitchen-terraform](https://github.com/newcontext-oss/kitchen-terraform), and [InSpec](https://github.com/inspec/inspec).
+
+Six test-kitchen instances are defined:
+
+- `deploy_service`
+- `node_pool`
+- `shared_vpc`
+- `simple_regional`
+- `simple_zonal`
+- `stub_domains`
+
+The test-kitchen instances in `test/fixtures/` wrap identically-named examples in the `examples/` directory.
+
+#### Setup
+
+1. Configure the [test fixtures](#test-configuration)
+2. Download a Service Account key with the necessary permissions and put it in the module's root directory with the name `credentials.json`.
+3. Build the Docker containers for testing:
+
+  ```
+  CREDENTIALS_FILE="credentials.json" make docker_build_terraform
+  CREDENTIALS_FILE="credentials.json" make docker_build_kitchen_terraform
+  ```
+4. Run the testing container in interactive mode:
+
+  ```
+  make docker_run
+  ```
+
+  The module root directory will be loaded into the Docker container at `/cftk/workdir/`.
+5. Run kitchen-terraform to test the infrastructure:
+
+  1. `kitchen create` creates Terraform state and downloads modules, if applicable.
+  2. `kitchen converge` creates the underlying resources. Run `kitchen converge <INSTANCE_NAME>` to create resources for a specific test case.
+  3. `kitchen verify` tests the created infrastructure. Run `kitchen verify <INSTANCE_NAME>` to run a specific test case.
+  4. `kitchen destroy` tears down the underlying resources created by `kitchen converge`. Run `kitchen destroy <INSTANCE_NAME>` to tear down resources for a specific test case.
+
+Alternatively, you can simply run `CREDENTIALS_FILE="credentials.json"  make test_integration_docker` to run all the test steps non-interactively.
+
+#### Test configuration
+
+Each test-kitchen instance is configured with a `variables.tfvars` file in the test fixture directory, e.g. `test/fixtures/node_pool/terraform.tfvars`.
+For convenience, since all of the variables are project-specific, these files have been symlinked to `test/fixtures/shared/terraform.tfvars`. 
+Similarly, each test fixture has a `variables.tf` to define these variables, and an `outputs.tf` to facilitate providing necessary information for `inspec` to locate and query against created resources.
+
+Each test-kitchen instance creates a GCP Network and Subnetwork fixture to house resources, and may create any other necessary fixture data as needed.
+
+### Autogeneration of documentation from .tf files
+Run
+```
+make generate_docs
+```
 
 ### Linting
 The makefile in this project will lint or sometimes just format any shell,