diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb9e180 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +**/.terraform/** +**tfstate* +**/*.pem +**/account.json +**/credentials +.DS_Store +.kitchen +.terraform +.terraform.tfstate.d +*.zip +# JetBrains - PyCharm, IntelliJ, etc. +.idea/ +__pycache__/ +*.iml +*.json +# Python +*.pyc diff --git a/.kitchen.yml b/.kitchen.yml new file mode 100644 index 0000000..1308729 --- /dev/null +++ b/.kitchen.yml @@ -0,0 +1,19 @@ +driver: + name: terraform + +provisioner: + name: terraform + +platforms: + - name: gcp + +verifier: + name: terraform + systems: + - name: fetch-secret + backend: local + +suites: + - name: "fetch-secret" + driver: + root_module_directory: test/fixtures/fetch-secret diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..2fffe26 --- /dev/null +++ b/Gemfile @@ -0,0 +1,21 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ruby "~> 2.5" + +source 'https://rubygems.org/' do + gem "kitchen-terraform", "~> 4.0" + gem "kubeclient", "~> 4.0" + gem "rest-client", "~> 2.0" +end diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..970097d --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,311 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) + aws-sdk (2.11.197) + aws-sdk-resources (= 2.11.197) + aws-sdk-core (2.11.197) + aws-sigv4 (~> 1.0) + jmespath (~> 1.0) + aws-sdk-resources (2.11.197) + aws-sdk-core (= 2.11.197) + aws-sigv4 (1.0.3) + azure_graph_rbac (0.17.0) + ms_rest_azure (~> 0.11.0) + azure_mgmt_key_vault (0.17.2) + ms_rest_azure (~> 0.11.0) + azure_mgmt_resources (0.17.2) + ms_rest_azure (~> 0.11.0) + builder (3.2.3) + coderay (1.1.2) + concurrent-ruby (1.1.4) + declarative (0.0.10) + declarative-option (0.1.0) + diff-lcs (1.3) + docker-api (1.34.2) + excon (>= 0.47.0) + multi_json + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + dry-configurable (0.7.0) + concurrent-ruby (~> 1.0) + dry-container (0.6.0) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (0.4.7) + concurrent-ruby (~> 1.0) + dry-equalizer (0.2.1) + dry-inflector (0.1.2) + dry-logic (0.4.2) + dry-container (~> 0.2, >= 0.2.6) + dry-core (~> 0.2) + dry-equalizer (~> 0.2) + dry-types (0.13.4) + concurrent-ruby (~> 1.0) + dry-container (~> 0.3) + dry-core (~> 0.4, >= 0.4.4) + dry-equalizer (~> 0.2) + dry-inflector (~> 0.1, >= 0.1.2) + dry-logic (~> 0.4, >= 0.4.2) + dry-validation (0.12.2) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (~> 0.2, >= 0.2.1) + dry-equalizer (~> 0.2) + dry-logic (~> 0.4, >= 0.4.0) + dry-types (~> 0.13.1) + equatable (0.5.0) + erubis (2.7.0) + excon (0.62.0) + faraday (0.15.4) + multipart-post (>= 1.2, < 3) + faraday-cookie_jar (0.0.6) + faraday (>= 0.7.4) + http-cookie (~> 1.0.0) + faraday_middleware (0.12.2) + faraday (>= 0.7.4, < 1.0) + ffi (1.9.25) + google-api-client (0.23.9) + addressable (~> 2.5, >= 2.5.1) + googleauth (>= 0.5, < 0.7.0) + httpclient (>= 2.8.1, < 3.0) + mime-types (~> 3.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.0) + signet (~> 0.9) + googleauth (0.6.7) + faraday (~> 0.12) + jwt (>= 1.4, < 3.0) + memoist (~> 0.16) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (~> 0.7) + gssapi (1.2.0) + ffi (>= 1.0.1) + gyoku (1.3.1) + builder (>= 2.1.2) + hashie (3.6.0) + htmlentities (4.3.4) + http (3.3.0) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 2.0) + http_parser.rb (~> 0.6.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http-form_data (2.1.1) + http_parser.rb (0.6.0) + httpclient (2.8.3) + inifile (3.0.0) + inspec (3.1.3) + addressable (~> 2.4) + faraday (>= 0.9.0) + faraday_middleware (~> 0.12.2) + hashie (~> 3.4) + htmlentities + json (>= 1.8, < 3.0) + method_source (~> 0.8) + mixlib-log + multipart-post + parallel (~> 1.9) + parslet (~> 1.5) + pry (~> 0) + rspec (~> 3) + rspec-its (~> 1.2) + rubyzip (~> 1.2, >= 1.2.2) + semverse + sslshake (~> 1.2) + term-ansicolor + thor (~> 0.20) + tomlrb (~> 1.2) + train (~> 1.5, >= 1.5.11) + tty-prompt (~> 0.17) + tty-table (~> 0.10) + jmespath (1.4.0) + json (2.1.0) + jwt (2.1.0) + kitchen-terraform (4.1.1) + dry-types (~> 0.9) + dry-validation (~> 0.10) + inspec (~> 3.0) + mixlib-shellout (~> 2.2) + test-kitchen (~> 1.23) + kubeclient (4.2.1) + http (~> 3.0) + recursive-open-struct (~> 1.0, >= 1.0.4) + rest-client (~> 2.0) + little-plugger (1.1.4) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) + memoist (0.16.0) + method_source (0.9.2) + mime-types (3.2.2) + mime-types-data (~> 3.2015) + mime-types-data (3.2018.0812) + mixlib-install (3.11.5) + mixlib-shellout + mixlib-versioning + thor + mixlib-log (2.0.9) + mixlib-shellout (2.4.4) + mixlib-versioning (1.2.7) + ms_rest (0.7.3) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + timeliness (~> 0.3) + ms_rest_azure (0.11.0) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + faraday-cookie_jar (~> 0.0.6) + ms_rest (~> 0.7.2) + multi_json (1.13.1) + multipart-post (2.0.0) + necromancer (0.4.0) + net-scp (1.2.1) + net-ssh (>= 2.6.5) + net-ssh (4.2.0) + net-ssh-gateway (1.3.0) + net-ssh (>= 2.6.5) + netrc (0.11.0) + nori (2.6.0) + os (1.0.0) + parallel (1.12.1) + parslet (1.8.2) + pastel (0.7.2) + equatable (~> 0.5.0) + tty-color (~> 0.4.0) + pry (0.12.2) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + public_suffix (3.0.3) + recursive-open-struct (1.1.0) + representable (3.0.4) + declarative (< 0.1.0) + declarative-option (< 0.2.0) + uber (< 0.2.0) + rest-client (2.0.2) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + retriable (3.1.2) + rspec (3.8.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-core (3.8.0) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-its (1.2.0) + rspec-core (>= 3.0.0) + rspec-expectations (>= 3.0.0) + rspec-mocks (3.8.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-support (3.8.0) + rubyntlm (0.6.2) + rubyzip (1.2.2) + semverse (3.0.0) + signet (0.11.0) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) + sslshake (1.2.0) + strings (0.1.4) + strings-ansi (~> 0.1.0) + unicode-display_width (~> 1.4.0) + unicode_utils (~> 1.4.0) + strings-ansi (0.1.0) + term-ansicolor (1.7.0) + tins (~> 1.0) + test-kitchen (1.23.5) + mixlib-install (~> 3.6) + mixlib-shellout (>= 1.2, < 3.0) + net-scp (~> 1.1) + net-ssh (>= 2.9, < 5.0) + net-ssh-gateway (~> 1.2) + thor (~> 0.19) + winrm (~> 2.0) + winrm-elevated (~> 1.0) + winrm-fs (~> 1.1) + thor (0.20.3) + timeliness (0.3.8) + timers (4.2.0) + tins (1.20.2) + tomlrb (1.2.8) + train (1.6.3) + aws-sdk (~> 2) + azure_graph_rbac (~> 0.16) + azure_mgmt_key_vault (~> 0.17) + azure_mgmt_resources (~> 0.15) + docker-api (~> 1.26) + google-api-client (~> 0.23.9) + googleauth (~> 0.6.6) + inifile + json (>= 1.8, < 3.0) + mixlib-shellout (~> 2.0) + net-scp (~> 1.2) + net-ssh (>= 2.9, < 6.0) + winrm (~> 2.0) + winrm-fs (~> 1.0) + tty-color (0.4.3) + tty-cursor (0.6.0) + tty-prompt (0.18.0) + necromancer (~> 0.4.0) + pastel (~> 0.7.0) + timers (~> 4.0) + tty-cursor (~> 0.6.0) + tty-reader (~> 0.5.0) + tty-reader (0.5.0) + tty-cursor (~> 0.6.0) + tty-screen (~> 0.6.4) + wisper (~> 2.0.0) + tty-screen (0.6.5) + tty-table (0.10.0) + equatable (~> 0.5.0) + necromancer (~> 0.4.0) + pastel (~> 0.7.2) + strings (~> 0.1.0) + tty-screen (~> 0.6.4) + uber (0.1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + unicode-display_width (1.4.1) + unicode_utils (1.4.0) + winrm (2.3.1) + builder (>= 2.1.2) + erubis (~> 2.7) + gssapi (~> 1.2) + gyoku (~> 1.0) + httpclient (~> 2.2, >= 2.2.0.2) + logging (>= 1.6.1, < 3.0) + nori (~> 2.0) + rubyntlm (~> 0.6.0, >= 0.6.1) + winrm-elevated (1.1.1) + winrm (~> 2.0) + winrm-fs (~> 1.0) + winrm-fs (1.3.2) + erubis (~> 2.7) + logging (>= 1.6.1, < 3.0) + rubyzip (~> 1.1) + winrm (~> 2.0) + wisper (2.0.0) + +PLATFORMS + ruby + +DEPENDENCIES + kitchen-terraform (~> 4.0)! + kubeclient (~> 4.0)! + rest-client (~> 2.0)! + +RUBY VERSION + ruby 2.5.1p57 + +BUNDLED WITH + 1.17.3 diff --git a/LICENSE b/LICENSE index 261eeb9..7a4a3ea 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -198,4 +199,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. + limitations under the License. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cbb1a3c --- /dev/null +++ b/Makefile @@ -0,0 +1,222 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make will use bash instead of sh +SHELL := /usr/bin/env bash + +# Docker build config variables +BUILD_TERRAFORM_VERSION ?= 0.11.10 +BUILD_CLOUD_SDK_VERSION ?= 216.0.0 +BUILD_PROVIDER_GOOGLE_VERSION ?= 1.17.1 +BUILD_PROVIDER_GSUITE_VERSION ?= 0.1.8 +DOCKER_IMAGE_TERRAFORM := cftk/terraform +DOCKER_TAG_TERRAFORM ?= ${BUILD_TERRAFORM_VERSION}_${BUILD_CLOUD_SDK_VERSION}_${BUILD_PROVIDER_GOOGLE_VERSION}_${BUILD_PROVIDER_GSUITE_VERSION} +BUILD_RUBY_VERSION := 2.5.3 +DOCKER_IMAGE_KITCHEN_TERRAFORM := cftk/kitchen_terraform +DOCKER_TAG_KITCHEN_TERRAFORM ?= ${BUILD_TERRAFORM_VERSION}_${BUILD_CLOUD_SDK_VERSION}_${BUILD_PROVIDER_GOOGLE_VERSION}_${BUILD_PROVIDER_GSUITE_VERSION} + +all: check_shell check_python check_golang check_terraform check_docker check_base_files test_check_headers check_headers check_trailing_whitespace generate_docs ## Run all linters and update documentation + +# The .PHONY directive tells make that this isn't a real target and so +# the presence of a file named 'check_shell' won't cause this target to stop +# working +.PHONY: check_shell +check_shell: ## Lint shell scripts + @source test/make.sh && check_shell + +.PHONY: check_python +check_python: ## Lint Python source files + @source test/make.sh && check_python + +.PHONY: check_golang +check_golang: ## Lint Go source files + @source test/make.sh && golang + +.PHONY: check_terraform +check_terraform: + @source ## Lint Terraform source files + +.PHONY: check_docker +check_docker: ## Lint Dockerfiles + @source test/make.sh && docker + +.PHONY: check_base_files +check_base_files: + @source test/make.sh && basefiles + +.PHONY: check_shebangs +check_shebangs: ## Check that scripts have correct shebangs + @source test/make.sh && check_bash + +.PHONY: check_trailing_whitespace +check_trailing_whitespace: + @source test/make.sh && check_trailing_whitespace + +.PHONY: test_check_headers +test_check_headers: + @echo "Testing the validity of the header check" + @python test/test_verify_boilerplate.py + +.PHONY: check_headers +check_headers: ## Check that source files have appropriate boilerplate + @echo "Checking file headers" + @python test/verify_boilerplate.py + +# Integration tests +.PHONY: test_integration +test_integration: test_integration_gcs_object test_integration_secret_infrastructure ## Run integration tests + bundle install + bundle exec kitchen create + # Yes, this is fugly. But we can fix this with terraform v0.12 when it is released + -bundle exec kitchen converge + bundle exec kitchen converge + bundle exec kitchen verify + bundle exec kitchen destroy + +.PHONY: test_integration_gcs_object +test_integration_gcs_object: ## Run integration tests + cd modules/gcs-object ; \ + bundle install ; \ + bundle exec kitchen create ; \ + bundle exec kitchen converge ; \ + bundle exec kitchen converge ; \ + bundle exec kitchen verify ; \ + bundle exec kitchen destroy ; \ + cd ../../ + +.PHONY: test_integration_secret_infrastructure +test_integration_secret_infrastructure: ## Run integration tests + cd modules/secret-infrastructure ; \ + bundle install ; \ + bundle exec kitchen create ; \ + bundle exec kitchen converge ; \ + bundle exec kitchen converge ; \ + bundle exec kitchen verify ; \ + bundle exec kitchen destroy ; \ + cd ../../ + + + + +.PHONY: generate_docs +generate_docs: ## Update README documentation for Terraform variables and outputs + @source test/make.sh && generate_docs + +.PHONY: release-new-version +release-new-version: + @source helpers/release-new-version.sh + +# Build Docker +.PHONY: docker_build_terraform +docker_build_terraform: + cd ${WORKDIR} ;\ + docker build -f ${CURDIR}/build/docker/terraform/Dockerfile \ + --build-arg BUILD_TERRAFORM_VERSION=${BUILD_TERRAFORM_VERSION} \ + --build-arg BUILD_CLOUD_SDK_VERSION=${BUILD_CLOUD_SDK_VERSION} \ + --build-arg BUILD_PROVIDER_GOOGLE_VERSION=${BUILD_PROVIDER_GOOGLE_VERSION} \ + --build-arg BUILD_PROVIDER_GSUITE_VERSION=${BUILD_PROVIDER_GSUITE_VERSION} \ + --build-arg CREDENTIALS_FILE=${CREDENTIALS_FILE} \ + -t ${DOCKER_IMAGE_TERRAFORM}:${DOCKER_TAG_TERRAFORM} . ;\ + cd ${CURDIR} +.PHONY: docker_build_kitchen_terraform +docker_build_kitchen_terraform: + cd ${WORKDIR} ;\ + docker build -f ${CURDIR}/build/docker/kitchen_terraform/Dockerfile \ + --build-arg BUILD_TERRAFORM_IMAGE="${DOCKER_IMAGE_TERRAFORM}:${DOCKER_TAG_TERRAFORM}" \ + --build-arg BUILD_RUBY_VERSION="${BUILD_RUBY_VERSION}" \ + --build-arg CREDENTIALS_FILE="${CREDENTIALS_FILE}" \ + -t ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} . ;\ + cd ${CURDIR} + +# Run docker +.PHONY: docker_run +docker_run: + docker run --rm -it \ + -v $(CURDIR):/cftk/workdir \ + ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ + /bin/bash + +.PHONY: docker_create +docker_create: docker_build_terraform docker_build_kitchen_terraform + docker run --rm -it \ + -v $(CURDIR):/cftk/workdir \ + ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ + /bin/bash -c "kitchen create" + +.PHONY: docker_converge +docker_converge: + docker run --rm -it \ + -v $(CURDIR):/cftk/workdir \ + ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ + /bin/bash -c "kitchen converge || kitchen converge" + +.PHONY: docker_verify +docker_verify: + docker run --rm -it \ + -v $(CURDIR):/cftk/workdir \ + ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ + /bin/bash -c "kitchen verify" + +.PHONY: docker_destroy +docker_destroy: + docker run --rm -it \ + -v $(CURDIR):/cftk/workdir \ + ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ + /bin/bash -c "kitchen destroy" + +.PHONY: test_integration_docker +test_integration_docker: WORKDIR = ${CURDIR} +test_integration_docker: docker_create docker_converge docker_verify docker_destroy test_integration_docker_gcs_object test_integration_docker_secret_infrastructure + @echo "Running test-kitchen tests in docker" + +.PHONY: test_integration_docker_gcs_object +test_integration_docker_gcs_object: WORKDIR = "./modules/gcs-object" +test_integration_docker_gcs_object: docker_create docker_converge docker_verify docker_destroy + @echo "Running test-kitchen tests for gcs-object in docker" + +.PHONY: test_integration_docker_secret_infrastructure +test_integration_docker_secret_infrastructure: WORKDIR = "./modules/secret-infrastructure" +test_integration_docker_secret_infrastructure: docker_create docker_converge docker_verify docker_destroy + @echo "Running test-kitchen tests for secret-infrastructure in docker" + + +help: ## Prints help for targets with comments + @grep -E '^[a-zA-Z._-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +#.PHONY: test_integration check-env +#.ONESHELL: +#test_integration: check-env ## Run a full integration test cycle +# @echo "Copying service-account-credentials.json to test dirs" +# cp service-account-credentials.json gcs-object/. +# cp service-account-credentials.json secret-infrastructure/. +# @echo Creating random string +# @echo "Running gcs-object integration test" +# cd gcs-object +# docker build . -f Dockerfile -t ubuntu-test-kitchen-terraform --build-arg RANDOM_SUFFIX=$(shell openssl rand -hex 5) --build-arg PROJECT_NAME=${PROJECT_NAME} --build-arg GOOGLE_APPLICATION_CREDENTIALS=service-account-credentials.json +# cd .. +# @echo "Running secret-infrastructure integration test" +# cd secret-infrastructure +# docker build . -f Dockerfile -t ubuntu-test-kitchen-terraform --build-arg RANDOM_SUFFIX=$(shell openssl rand -hex 5) --build-arg PROJECT_NAME=${PROJECT_NAME} --build-arg GOOGLE_APPLICATION_CREDENTIALS=service-account-credentials.json +# cd .. +# +# @echo "Running overall test-kitchen in docker" +# docker build . -f Dockerfile -t ubuntu-test-kitchen-terraform --build-arg RANDOM_SUFFIX=$(shell openssl rand -hex 5) --build-arg PROJECT_NAME=${PROJECT_NAME} --build-arg GOOGLE_APPLICATION_CREDENTIALS=service-account-credentials.json +# +#help: ## Prints help for targets with comments +# @grep -E '^[a-zA-Z._-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +# +#check-env: +#ifndef PROJECT_NAME +# $(error PROJECT_NAME is undefined) +#endif \ No newline at end of file diff --git a/README.md b/README.md index 6684e5d..cbd45ed 100644 --- a/README.md +++ b/README.md @@ -1 +1,278 @@ -# terraform-google-secret \ No newline at end of file +# terraform-google-secret + +# Google Cloud GCS based secret management Terraform Module + +This Terraform module makes it easier to manage to manage secrets for your Google Cloud environment, such as api keys, tokens, etc. + +Specifically, this repo provides modules to store secrets in app specific GCS buckets, shared buckets, and to fetch the secrets as needed. +Note this is all on a per environment basis as well. + +## Usage +Examples are included in the [examples](./examples/) folder. + +There are two key operations, creating the buckets, and fetching a secret. +Note that setting/storing a secret is a separate and out-of-band process. See [this readme for more information.](./modules/secret-infrastructure/infra/README.md) + +The base module is to fetch a secret from already created buckets. Submodules are for fetching a file from GCS, and creating the buckets. +The GCS fetching submodule is located at [./modules/secret-infrastructure](./modules/secret-infrastructure) +The bucket creation submodule is located at [./modules/secret-infrastructure](./modules/secret-infrastructure) + +### Fetching a secret +A simple example to fetch a secret is as follows: + +```hcl +module "fetch-secret" { + source = "github.com/terraform-google-modules/terraform-google-secret" + env = "dev" + application_name = "app1" + secret = "api-key" + credentials_file_path = "sservice-account-credentials.json" +} +``` + +Creating the bucket infrastructure is a submodule in this repo. + +### Creating the buckets + +A simple example to create buckets is as follows: + +```hcl +module "secret-storage" { + source = "github.com/terraform-google-modules/terraform-google-secret/modules/secret-infrastructure" + project_name = "your-secret-storage-project" + application_list = ["webapp", "service1"] + env_list = ["dev", "qa", "production"] + credentials_file_path = "service-account-creds.json" +} +``` +This will create buckets with of the form: appname-env-secrets +Using the above as an example, this will create: +* webapp-dev-secrets +* webapp-qa-secrets +* webapp-prod-secrets +* service1-dev-secrets +* service1-qa-secrets +* service1-prod-secrets + +Along with the shared buckets per environment +* shared-projectname-dev-secrets +* shared-projectname-qa-secrets +* shared-projectname-prod-secrets + +Specific submodule docs can be found [in the submodule](secret-infrastructure/README.md) + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| application\_name | The application to fetch secrets for | string | - | yes | +| credentials\_file\_path | The path to the GCP credentials | string | - | yes | +| env | The environment to fetch secrets for | string | - | yes | +| secret | The name of the secret to fetch | string | - | yes | +| shared | Will we fetch the secret from the shared bucket instead of an application-specific bucket? | string | `false` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| contents | The contents of the requested GCS object | + +[^]: (autogen_docs_end) + +## Requirements +### Terraform plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.10.x +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v1.10.0 + +## Install + +### Terraform +Be sure you have the correct Terraform version (0.10.x), you can choose the binary here: +- https://releases.hashicorp.com/terraform/ + +### Terraform plugins + +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v1.10.0 + + +## Development + +### File structure +The project has the following folders and files: + +- /: root folder +- /examples: examples for using this module +- /test: Folders with files for testing the module (see Testing section on this file) +- /main.tf: main file for this module, contains primary logic for operate the module +- /variables.tf: all the variables for the module +- /output.tf: the outputs of the module +- /readme.MD: this file +- /modules: submodules. See individual modules for README's. + +## Testing + +### Requirements +- [bundler](https://github.com/bundler/bundler) +- [gcloud](https://cloud.google.com/sdk/install) +- [terraform-docs](https://github.com/segmentio/terraform-docs/releases) 0.3.0 + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Integration test + +Integration tests are run though [test-kitchen](https://github.com/test-kitchen/test-kitchen), [kitchen-terraform](https://github.com/newcontext-oss/kitchen-terraform), and [InSpec](https://github.com/inspec/inspec). + +#### Setup + +1. Configure the [test fixtures](#test-configuration) +2. Download a Service Account key with the necessary permissions and put it in the module's root directory with the name `credentials.json`. +3. Build the Docker containers for testing: + + ``` + CREDENTIALS_FILE="credentials.json" make docker_build_terraform + CREDENTIALS_FILE="credentials.json" make docker_build_kitchen_terraform + ``` +4. Run the testing container in interactive mode: + + ``` + make docker_run + ``` + + The module root directory will be loaded into the Docker container at `/cftk/workdir/`. +5. Run kitchen-terraform to test the infrastructure: + + 1. `kitchen create` creates Terraform state and downloads modules, if applicable. + 2. `kitchen converge` creates the underlying resources. Run `kitchen converge ` to create resources for a specific test case. + 3. `kitchen verify` tests the created infrastructure. Run `kitchen verify ` to run a specific test case. + 4. `kitchen destroy` tears down the underlying resources created by `kitchen converge`. Run `kitchen destroy ` to tear down resources for a specific test case. + +Alternatively, you can simply run `CREDENTIALS_FILE="credentials.json" make test_integration_docker` to run all the test steps non-interactively. + +#### Test configuration + +Each test-kitchen instance is configured with a `terraform.tfvars` file in the test fixture directory, e.g. `test/fixtures/fetch-secret/terraform.tfvars`. +Similarly, each test fixture has a `variables.tf` to define these variables, and an `outputs.tf` to facilitate providing necessary information for `inspec` to locate and query against created resources. + +For running the test fixture in docker `make test_integration_docker`, set the variable `credentials_file_path` to your filename as above, but with the path as follows +`credentials_file_path="/cftk/workdir/credentials.json` +`` + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Linting +The makefile in this project will lint or sometimes just format any shell, +Python, golang, Terraform, or Dockerfiles. The linters will only be run if +the makefile finds files with the appropriate file extension. + +All of the linter checks are in the default make target, so you just have to +run + +``` +make -s +``` + +The -s is for 'silent'. Successful output looks like this, though there are currently failing files. + +``` +Running shellcheck +Running flake8 +Running gofmt +Running terraform validate +Running hadolint on Dockerfiles +Test passed - Verified all file Apache 2 headers +``` + +The linters +are as follows: +* Shell - shellcheck. Can be found in homebrew +* Golang - gofmt. gofmt comes with the standard golang installation. golang +is a compiled language so there is no standard linter. +* Terraform - terraform has a built-in linter in the 'terraform validate' +command. +* Dockerfiles - hadolint. Can be found in homebrew + + +## Setting secrets concepts +In general, this repo is built to run in CI/CD. It's not usually appropriate to have the setting of secrets performed in that manner. + +The helpers/ directory has a basic concept for setting secrets. The code is here doesn't run as-is (bucket names will need to be adjusted, etc), but is provided as a way to think about setting secrets. +Conceptually, secrets are set manually by a team authorized to perform this work. + +The team would use the `helpers/set-secrets.sh` script to set the secret (see [below](#setting-secrets)). + +In this scenario, the GCS buckets are the source of truth. It might be necessary to have a list of the secrets stored in source control for auditability. See [jenkins](#jenkins-automation). + +### File structure +The helpers directory has the following folders and files: + +- helpers/ - Contains the helper scripts for setting/clearing scripts (set/list/clear-secret.sh, set-secrets.sh) +- helpers/jenkins - Contains an example Jenkins Groovy for capturing defined secret names in source + + +### Setting Secrets +Application secrets are set using the following inputs: + +- APPLICATION_NAME (ex. `app1`) +- ENVIRONMENT (ex. `dev`) +- SECRET_NAME is the key the secret will be referenced by (ex. `my-secret`) +- SECRET_VALUE is the value of the secret (ex. `my-secret-value`) + +The script is executed like this: + +``` +./helpers/set-secret.sh APPLICATION_NAME ENVIRONMENT SECRET_NAME SECRET_VALUE +``` + +For example: +``` +./helpers/set-secret.sh app1 dev my-secret my-secret-name +``` + +Shared secrets which are referenced by multiple applications are set the same way, simply using `shared` as the application name. For example: + +``` +./helpers/set-secret.sh shared dev datadog-key the-datadog-secret-key +``` + +Multiple secrets can be set at once using the [`set-secrets.sh`](./helpers/set-secrets.sh) script, which accepts a JSON file (syntax example in [`secrets.json`](./examples/secrets.json)). + +``` +./helpers/set-secrets.sh examples/secrets.json +``` + +### Clearing Secrets +Secrets which are no longer needed should be cleared using the secret clearing sript: + +``` +./helpers/clear-secret.sh APPLICATION_NAME ENVIRONMENT SECRET_NAME +``` + +### Listing Secrets +A listing of all secrets managed by this module is generated with this command: + +``` +./helpers/list-secrets.sh +``` + +### Jenkins automation + +[./helpers/jenkins](./helpers/jenkins) is a pipeline with the goal of having an up to date capture of the defined secrets in source. The groovy script is triggered in Jenkins (via cron). +This script calls [list-secrets.sh](./helpers/list-secrets.sh), which fetches all the defined secrets. It then calls [commit-list.sh](./helpers/jenkins/commit-list.sh) to commit them to source control. All three of these files will need modification to work correctly in your environments. + +``` +./helpers/jenkins/default-update-secrets-list.groovy +``` + +``` +./helpers/jenkins/commit-list.sh +``` diff --git a/build/docker/kitchen_terraform/Dockerfile b/build/docker/kitchen_terraform/Dockerfile new file mode 100644 index 0000000..c021b0c --- /dev/null +++ b/build/docker/kitchen_terraform/Dockerfile @@ -0,0 +1,69 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BUILD_TERRAFORM_IMAGE +ARG BUILD_RUBY_VERSION +# hadolint ignore=DL3006 +FROM $BUILD_TERRAFORM_IMAGE as cfkt_terraform + + + +FROM ruby:$BUILD_RUBY_VERSION-alpine + +RUN apk add --no-cache \ + bash=4.4.19-r1 \ + curl=7.61.1-r1 \ + git=2.18.1-r0 \ + g++=6.4.0-r9 \ + jq=1.6_rc1-r1 \ + make=4.2.1-r2 \ + musl-dev=1.1.19-r10 \ + python2=2.7.15-r1 \ + python2-dev=2.7.15-r1 \ + py2-pip=10.0.1-r0 \ + ca-certificates=20171114-r3 + +ADD https://storage.googleapis.com/kubernetes-release/release/v1.12.2/bin/linux/amd64/kubectl /usr/local/bin/kubectl +RUN chmod +x /usr/local/bin/kubectl + +SHELL ["/bin/bash", "-c"] + +ENV APP_BASE_DIR="/cftk" + +COPY --from=cfkt_terraform $APP_BASE_DIR $APP_BASE_DIR + +ARG CREDENTIALS_FILE + +ENV HOME="$APP_BASE_DIR/home" +ENV PATH $APP_BASE_DIR/bin:$APP_BASE_DIR/google-cloud-sdk/bin:$PATH +ENV GOOGLE_APPLICATION_CREDENTIALS="$APP_BASE_DIR/workdir/$CREDENTIALS_FILE" \ + CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$APP_BASE_DIR/workdir/$CREDENTIALS_FILE" + +# Fix base64 inconsistency +SHELL ["/bin/bash", "-c"] +RUN echo 'base64() { if [[ $@ == "--decode" ]]; then command base64 -d | more; else command base64 "$@"; fi; }' >> $APP_BASE_DIR/home/.bashrc + +RUN terraform --version && \ + gcloud --version && \ + ruby --version && \ + bundle --version + +COPY ./Gemfile /opt/kitchen/ + +WORKDIR /opt/kitchen +RUN bundle install + +RUN gcloud components install beta --quiet + +WORKDIR $APP_BASE_DIR/workdir diff --git a/build/docker/terraform/Dockerfile b/build/docker/terraform/Dockerfile new file mode 100644 index 0000000..fed6548 --- /dev/null +++ b/build/docker/terraform/Dockerfile @@ -0,0 +1,102 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine:3.8 as builder + +RUN apk add --no-cache \ + bash=4.4.19-r1 \ + git=2.18.1-r0 \ + go=1.10.7-r0 \ + make=4.2.1-r2 \ + musl-dev=1.1.19-r10 + +ENV APP_BASE_DIR="/cftk" + +RUN mkdir -p $APP_BASE_DIR/home && \ + mkdir -p $APP_BASE_DIR/bin && \ + mkdir -p $APP_BASE_DIR/workdir + +ENV GOPATH="/root/go" + +ARG BUILD_PROVIDER_GOOGLE_VERSION +ENV PROVIDER_GOOGLE_VERSION="${BUILD_PROVIDER_GOOGLE_VERSION}" + +RUN mkdir -p $APP_BASE_DIR/home/.terraform.d/plugins && \ + mkdir -p $GOPATH/src/github.com/terraform-providers && \ + git clone https://github.com/terraform-providers/terraform-provider-google.git $GOPATH/src/github.com/terraform-providers/terraform-provider-google + +WORKDIR $GOPATH/src/github.com/terraform-providers/terraform-provider-google +RUN git fetch --all --tags --prune && \ + git checkout tags/v${PROVIDER_GOOGLE_VERSION} -b v${PROVIDER_GOOGLE_VERSION} && \ + make build && \ + mv $GOPATH/bin/terraform-provider-google $APP_BASE_DIR/home/.terraform.d/plugins + + + +FROM alpine:3.8 + +RUN apk add --no-cache \ + bash=4.4.19-r1 \ + curl=7.61.1-r1 \ + git=2.18.1-r0 \ + jq=1.6_rc1-r1 \ + make=4.2.1-r2 \ + python2=2.7.15-r1 + +ARG CREDENTIALS_FILE + +ENV APP_BASE_DIR="/cftk" + +COPY --from=builder $APP_BASE_DIR $APP_BASE_DIR + +ENV HOME="$APP_BASE_DIR/home" +ENV PATH $APP_BASE_DIR/bin:$APP_BASE_DIR/google-cloud-sdk/bin:$PATH +ENV GOOGLE_APPLICATION_CREDENTIALS="$APP_BASE_DIR/workdir/$CREDENTIALS_FILE" \ + CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$APP_BASE_DIR/workdir/$CREDENTIALS_FILE" + +# Fix base64 inconsistency +SHELL ["/bin/bash", "-c"] +RUN echo 'base64() { if [[ $@ == "--decode" ]]; then command base64 -d | more; else command base64 "$@"; fi; }' >> $APP_BASE_DIR/home/.bashrc + +ARG BUILD_CLOUD_SDK_VERSION +ENV CLOUD_SDK_VERSION="${BUILD_CLOUD_SDK_VERSION}" + +WORKDIR $APP_BASE_DIR +RUN curl -LO https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + ln -s /lib /lib64 && \ + gcloud config set core/disable_usage_reporting true && \ + gcloud config set component_manager/disable_update_check true && \ + gcloud config set metrics/environment github_docker_image && \ + gcloud --version + +ARG BUILD_TERRAFORM_VERSION +ENV TERRAFORM_VERSION="${BUILD_TERRAFORM_VERSION}" + +RUN curl -LO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + mv terraform $APP_BASE_DIR/bin && \ + terraform --version + +ARG BUILD_PROVIDER_GSUITE_VERSION +ENV PROVIDER_GSUITE_VERSION="${BUILD_PROVIDER_GSUITE_VERSION}" + +RUN curl -LO https://github.com/DeviaVir/terraform-provider-gsuite/releases/download/v${PROVIDER_GSUITE_VERSION}/terraform-provider-gsuite_${PROVIDER_GSUITE_VERSION}_linux_amd64.tgz && \ + tar xzf terraform-provider-gsuite_${PROVIDER_GSUITE_VERSION}_linux_amd64.tgz && \ + rm terraform-provider-gsuite_${PROVIDER_GSUITE_VERSION}_linux_amd64.tgz && \ + mv terraform-provider-gsuite_v${PROVIDER_GSUITE_VERSION} $APP_BASE_DIR/home/.terraform.d/plugins/terraform-provider-gsuite + +WORKDIR $APP_BASE_DIR/workdir diff --git a/examples/bucket_example/input_vars.tfvars b/examples/bucket_example/input_vars.tfvars new file mode 100644 index 0000000..087eb00 --- /dev/null +++ b/examples/bucket_example/input_vars.tfvars @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +application_list=["app1", "app2", "app3"] +env_list=["dev","qa","prod"] +project_name="test-project" +credentials_file_path = "/Users/rishimalik/code/project-factory-service-account-credentials.json" \ No newline at end of file diff --git a/examples/bucket_example/main.tf b/examples/bucket_example/main.tf new file mode 100644 index 0000000..7849e38 --- /dev/null +++ b/examples/bucket_example/main.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "secret-storage" { + source = "../../secret-infrastructure" + project_name = "${var.project_name}" + application_list = "${var.application_list}" + env_list = "${var.env_list}" + credentials_file_path = "${var.credentials_file_path}" +} diff --git a/examples/bucket_example/outputs.tf b/examples/bucket_example/outputs.tf new file mode 100644 index 0000000..ee50d7d --- /dev/null +++ b/examples/bucket_example/outputs.tf @@ -0,0 +1,16 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + diff --git a/examples/bucket_example/variables.tf b/examples/bucket_example/variables.tf new file mode 100644 index 0000000..fb336d2 --- /dev/null +++ b/examples/bucket_example/variables.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "application_list" { + description = "The list of application names that will store secrets" + type = "list" + default = [] +} + +variable "env_list" { + description = "The list of environments for secrets" + type = "list" + default = [] +} + +variable "project_name" { + description = "The name of the project this applies to" +} + +variable "credentials_file_path" { + description = "GCP credentials fils" +} diff --git a/examples/fetch_secret/input_vars.tfvars b/examples/fetch_secret/input_vars.tfvars new file mode 100644 index 0000000..79d269d --- /dev/null +++ b/examples/fetch_secret/input_vars.tfvars @@ -0,0 +1,21 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +application_name="test_application" +env="development" +secret="api_key" +shared="false" +credentials_file_path = "/Users/rishimalik/code/project-factory-service-account-credentials.json" \ No newline at end of file diff --git a/examples/fetch_secret/main.tf b/examples/fetch_secret/main.tf new file mode 100644 index 0000000..6785de3 --- /dev/null +++ b/examples/fetch_secret/main.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "mysecret" { + source = "../../" + env = "${var.env}" + application_name = "${var.application_name}" + secret = "${var.secret}" + credentials_file_path = "${var.credentials_file_path}" +} diff --git a/examples/fetch_secret/outputs.tf b/examples/fetch_secret/outputs.tf new file mode 100644 index 0000000..0c6a856 --- /dev/null +++ b/examples/fetch_secret/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "mysecret" { + value = "${module.mysecret.contents}" + sensitive = true +} diff --git a/examples/fetch_secret/variables.tf b/examples/fetch_secret/variables.tf new file mode 100644 index 0000000..ba440c6 --- /dev/null +++ b/examples/fetch_secret/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "application_name" { + description = "The application to fetch secrets for" +} + +variable "credentials_file_path" { + description = "GCP credentials fils" +} + +variable "env" { + description = "The environment to fetch secrets for" +} + +variable "secret" { + description = "The name of the secret to fetch" +} diff --git a/helpers/clear-secret.sh b/helpers/clear-secret.sh new file mode 100755 index 0000000..a1a881d --- /dev/null +++ b/helpers/clear-secret.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#description :This script is used to clear a secrets. Check README.md for full instructions. +#usage :bash ./scripts/clear-secret.sh $APPLICATION $ENVIRONMENT $SECRET_KEY +#============================================================================== + +# Parse params +die () { + echo >&2 "$@" + exit 1 +} + +[ "$#" -eq 3 ] || die "3 arguments required, $# provided" + +APPLICATION=$1 +ENVIRONMENT=$2 +SECRET_KEY=$3 + +# Build our bucket data +BUCKET_NAME="project-$APPLICATION-$ENVIRONMENT-secrets" +OBJECT_KEY="$SECRET_KEY.txt" + +# Set the bucket object data +gsutil rm "gs://$BUCKET_NAME/$OBJECT_KEY" diff --git a/helpers/combine_docfiles.py b/helpers/combine_docfiles.py new file mode 100755 index 0000000..5849d54 --- /dev/null +++ b/helpers/combine_docfiles.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' Combine file from: + * script argument 1 + with content of file from: + * script argument 2 + using the beginning of line separators + hardcoded using regexes in this file: + + We exclude any text using the separate + regex specified here +''' + +import re +import sys + +insert_separator_regex = '(.*?\[\^\]\:\ \(autogen_docs_start\))(.*?)(\n\[\^\]\:\ \(autogen_docs_end\).*?$)' +exclude_separator_regex = '(.*?)Copyright 20\d\d Google LLC.*?limitations under the License.(.*?)$' + +if len(sys.argv) != 3: + sys.exit(1) +try: + input = open(sys.argv[1], "r").read() +except IOError as e: + sys.exit(1) +replace_content = open(sys.argv[2], "r").read() + +# Exclude the specified content from the replacement content +groups = re.match(exclude_separator_regex, replace_content, re.DOTALL).groups(0) +replace_content = groups[0] + groups[1] + +# Find where to put the replacement content, overwrite the input file +try: + groups = re.match(insert_separator_regex, input, re.DOTALL).groups(0) + output = groups[0] + replace_content + groups[2] +except AttributeError as e: + sys.exit(1) + +open(sys.argv[1], "w").write(output) diff --git a/helpers/jenkins/commit-list.sh b/helpers/jenkins/commit-list.sh new file mode 100755 index 0000000..f09f3bb --- /dev/null +++ b/helpers/jenkins/commit-list.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Get absolute path of secrets directory +secrets_dir="$(git rev-parse --show-toplevel)/secrets" + +# Add changes to secrets/ +git add "$secrets_dir" || exit $? + +# Commit changes if any exist vs. HEAD +if [ -n "$(git diff-index HEAD)" ]; then + git config --global user.name "Automation" + git config --global user.email "test@example.com" + git commit -m "[Jenkins] Update secrets list" -m "${BUILD_URL}" "$secrets_dir" + git push -u origin master +else + echo "No changes to commit." + exit 0 +fi + diff --git a/helpers/jenkins/default-update-secrets-list.groovy b/helpers/jenkins/default-update-secrets-list.groovy new file mode 100644 index 0000000..b930787 --- /dev/null +++ b/helpers/jenkins/default-update-secrets-list.groovy @@ -0,0 +1,43 @@ +pipeline { + agent any + options { + // This will activate red and green text - turn this off for vision accessibility reasons + ansiColor('xterm') + // Prevent more than one of this pipeline from running at once + disableConcurrentBuilds() + } + triggers { + // Run the job hourly + cron('H * * * *') + } + environment { + // Service Account to use + SA = "PLACE SERVICE ACCOUNT HERE" + // Service Account credentials + } + stages { + stage('Update secrets') { + steps { + sh "${env.WORKSPACE}/infra/scripts/list-secrets.sh" + } + } + stage('Commit secrets') { + steps { + sh "${env.WORKSPACE}/infra/jenkins/commit-list.sh" + } + } + } + // Post describes the steps to take when the pipeline finishes + post { + //changed {} + //aborted {} + //failure {} + //success {} + //unstable {} + //notBuilt {} + always { + echo "Clearing workspace" + deleteDir() // Clean up the local workspace so we don't leave behind a mess, or sensitive files + } + } +} \ No newline at end of file diff --git a/helpers/list-secrets.sh b/helpers/list-secrets.sh new file mode 100755 index 0000000..95a1b97 --- /dev/null +++ b/helpers/list-secrets.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#description :This script is used to generate a list of secrets. +#usage :bash ./scripts/list-secrets.sh +#============================================================================== + +# Parse params +die () { + echo >&2 "$@" + exit 1 +} + +SECRETS_PROJECT="secrets" +ROOT_FOLDER="./secrets/" + +# Reset the secrets folder +rm -rf "$ROOT_FOLDER" + +echo "Generating list of secrets in $ROOT_FOLDER" + +BUCKETS=$(gsutil ls -p "$SECRETS_PROJECT" ) +[[ $? -ne 0 ]] && die 'ERROR: Failed to get buckets list' + +for BUCKET in $BUCKETS +do + echo "Listing secrets stored in $BUCKET" + + # Parse bucket to find app and env + [[ $BUCKET =~ gs://projects-(.*)-(.*)-secrets/ ]] + APP=${BASH_REMATCH[1]} + ENV=${BASH_REMATCH[2]} + FILE_PATH="$ROOT_FOLDER/$APP/$ENV.txt" + + # Make folder to hold secrets + mkdir -p "$(dirname "$FILE_PATH")" + + # Print contents of bucket + BUCKET_ESCAPED=$(echo "$BUCKET" | sed -e 's/[]\/$*.^[]/\\&/g') + bucket_ls=$(gsutil ls "$BUCKET") || die "ERROR: Failed to get objects list from $BUCKET" + echo "$bucket_ls" | sed "s/$BUCKET_ESCAPED\(.*\).txt/\1/" >> "$FILE_PATH" +done diff --git a/helpers/set-secret.sh b/helpers/set-secret.sh new file mode 100755 index 0000000..6d2a841 --- /dev/null +++ b/helpers/set-secret.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#description :This script is used to set secrets. Check README.md for full instructions. +#usage :bash ./scripts/set-secret.sh $APPLICATION $ENVIRONMENT $SECRET_KEY $SECRET_VALUE +#============================================================================== + +# Parse params +die () { + echo >&2 "$@" + exit 1 +} + +[ "$#" -eq 4 ] || die "4 arguments required, $# provided" + +APPLICATION=$1 +ENVIRONMENT=$2 +SECRET_KEY=$3 +SECRET_VALUE="$4" + +# Build our bucket data +BUCKET_NAME=project-$APPLICATION-$ENVIRONMENT-secrets +OBJECT_KEY="$SECRET_KEY.txt" + +# Set the bucket object data +echo -n "$SECRET_VALUE" | gsutil -h "Content-Type:text/plain" cp - "gs://$BUCKET_NAME/$OBJECT_KEY" diff --git a/helpers/set-secrets.sh b/helpers/set-secrets.sh new file mode 100755 index 0000000..93fda6e --- /dev/null +++ b/helpers/set-secrets.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#description :This script is used to set multiple secrets at once. Check README.md for full instructions. +#usage :bash ./scripts/set-secrets.sh $CONFIG.json +#============================================================================== + +# Parse params +die () { + echo >&2 "$@" + exit 1 +} + +[ "$#" -eq 1 ] || die "1 argument required, $# provided" + +JSON_FILE=$1 + +for app_object in $(< "${JSON_FILE}" jq -r '.[] | @base64'); do + _jq() { + echo "${app_object}" | base64 --decode | jq -r "${1}" + } + + APP_NAME=$(_jq '.app') + ENVS=$(_jq '.environments[]') +# SECRETS=$(_jq '.secrets[]') + + for SECRET_NAME in $(_jq '.secrets | keys[]'); do + SECRET_VALUE=$(_jq ".secrets.$SECRET_NAME") + for ENV in $ENVS; do + echo "Setting $SECRET_NAME for $APP_NAME in $ENV" + "${BASH_SOURCE%/*}"/set-secret.sh "$APP_NAME" "$ENV" "$SECRET_NAME" "$SECRET_VALUE" + done + done +done diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..ce4eae9 --- /dev/null +++ b/main.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************** + Provider configuration + *****************************************/ +provider "google" { + credentials = "${file(var.credentials_file_path)}" +} + +locals { + // secret_project = "${var.project_name}" + shared_bucket = "shared-${var.env}-secrets" + app_bucket = "${var.application_name}-${var.env}-secrets" + bucket_name = "${var.shared == "true" ? local.shared_bucket : local.app_bucket}" + object_path = "${var.secret}.txt" +} + +module "secret" { + source = "./modules/gcs-object" + bucket = "${local.bucket_name}" + path = "${local.object_path}" +} diff --git a/modules/gcs-object/.kitchen.yml b/modules/gcs-object/.kitchen.yml new file mode 100644 index 0000000..588ad2a --- /dev/null +++ b/modules/gcs-object/.kitchen.yml @@ -0,0 +1,20 @@ +--- +driver: + name: terraform + +provisioner: + name: terraform + +platforms: +- name: gcp + +verifier: + name: terraform + systems: + - name: bucket + backend: local + +suites: +- name: "barebones" + driver: + root_module_directory: examples/barebones-test-fixture \ No newline at end of file diff --git a/modules/gcs-object/CHANGELOG.md b/modules/gcs-object/CHANGELOG.md new file mode 100644 index 0000000..3f48666 --- /dev/null +++ b/modules/gcs-object/CHANGELOG.md @@ -0,0 +1,5 @@ +# Change Log + +### Added + +* Everything! Initial release of the module. diff --git a/modules/gcs-object/Dockerfile b/modules/gcs-object/Dockerfile new file mode 100644 index 0000000..f7a1612 --- /dev/null +++ b/modules/gcs-object/Dockerfile @@ -0,0 +1,54 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:bionic + +RUN apt-get update -y && \ + apt-get install -y software-properties-common && \ + apt-add-repository -y ppa:rael-gc/rvm && \ + apt-get update -y && \ + apt-get install rvm -y && \ + /bin/bash -l -c "rvm install 2.4.2 && \ + echo 'gem: --no-ri --no-rdoc' > ~/.gemrc && \ + gem install bundler --no-ri --no-rdoc" + +RUN apt-get install -y unzip wget ssh git && \ + wget https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip && \ + unzip terraform_0.11.11_linux_amd64.zip && \ + mv terraform /usr/local/bin/ + +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update -y && apt-get install google-cloud-sdk -y + +COPY Gemfile ./ +RUN /bin/bash -l -c "bundle install" + +ENV WORKDIR /root/static_build +WORKDIR $WORKDIR +COPY . . +ARG GOOGLE_APPLICATION_CREDENTIALS +ENV CREDENTIALS_FILE credentials.json +COPY $GOOGLE_APPLICATION_CREDENTIALS $WORKDIR/$CREDENTIALS_FILE +ENV GOOGLE_APPLICATION_CREDENTIALS=$WORKDIR/$CREDENTIALS_FILE + +RUN echo "alias tf_list=\"/bin/bash -l -c 'bundle exec kitchen list'\"" >> /root/.bashrc && \ + echo "alias tf_destroy=\"/bin/bash -l -c 'bundle exec kitchen destroy ; bundle exec kitchen list'\"" >> /root/.bashrc && \ + echo "alias tf_test=\"/bin/bash -l -c 'bundle exec kitchen create && bundle exec kitchen converge && bundle exec kitchen verify ; bundle exec kitchen list'\"" >> /root/.bashrc && \ + echo "alias tf_test_and_destroy=\"/bin/bash -l -c 'bundle exec kitchen create && bundle exec kitchen test --destroy always'\"" >> /root/.bashrc + +ARG PROJECT_NAME +ENV PROJECT_NAME=$PROJECT_NAME +RUN /bin/bash -l -c 'bundle exec kitchen create && bundle exec kitchen converge ; bundle exec kitchen converge && bundle exec kitchen verify && bundle exec kitchen destroy' \ No newline at end of file diff --git a/modules/gcs-object/Gemfile b/modules/gcs-object/Gemfile new file mode 100644 index 0000000..9f72397 --- /dev/null +++ b/modules/gcs-object/Gemfile @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +ruby '2.4.2' + +source 'https://rubygems.org/' do + gem 'inspec', '~> 3.1.0' + gem 'kitchen-terraform', '~> 4.1' + gem "kitchen-google", "~> 2.0" +end diff --git a/modules/gcs-object/Gemfile.lock b/modules/gcs-object/Gemfile.lock new file mode 100644 index 0000000..27bbd90 --- /dev/null +++ b/modules/gcs-object/Gemfile.lock @@ -0,0 +1,300 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) + aws-sdk (2.11.197) + aws-sdk-resources (= 2.11.197) + aws-sdk-core (2.11.197) + aws-sigv4 (~> 1.0) + jmespath (~> 1.0) + aws-sdk-resources (2.11.197) + aws-sdk-core (= 2.11.197) + aws-sigv4 (1.0.3) + azure_graph_rbac (0.17.0) + ms_rest_azure (~> 0.11.0) + azure_mgmt_key_vault (0.17.2) + ms_rest_azure (~> 0.11.0) + azure_mgmt_resources (0.17.2) + ms_rest_azure (~> 0.11.0) + builder (3.2.3) + coderay (1.1.2) + concurrent-ruby (1.1.4) + declarative (0.0.10) + declarative-option (0.1.0) + diff-lcs (1.3) + docker-api (1.34.2) + excon (>= 0.47.0) + multi_json + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + dry-configurable (0.7.0) + concurrent-ruby (~> 1.0) + dry-container (0.6.0) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (0.4.7) + concurrent-ruby (~> 1.0) + dry-equalizer (0.2.1) + dry-inflector (0.1.2) + dry-logic (0.4.2) + dry-container (~> 0.2, >= 0.2.6) + dry-core (~> 0.2) + dry-equalizer (~> 0.2) + dry-types (0.13.4) + concurrent-ruby (~> 1.0) + dry-container (~> 0.3) + dry-core (~> 0.4, >= 0.4.4) + dry-equalizer (~> 0.2) + dry-inflector (~> 0.1, >= 0.1.2) + dry-logic (~> 0.4, >= 0.4.2) + dry-validation (0.12.2) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (~> 0.2, >= 0.2.1) + dry-equalizer (~> 0.2) + dry-logic (~> 0.4, >= 0.4.0) + dry-types (~> 0.13.1) + equatable (0.5.0) + erubis (2.7.0) + excon (0.62.0) + faraday (0.15.4) + multipart-post (>= 1.2, < 3) + faraday-cookie_jar (0.0.6) + faraday (>= 0.7.4) + http-cookie (~> 1.0.0) + faraday_middleware (0.12.2) + faraday (>= 0.7.4, < 1.0) + ffi (1.9.25) + gcewinpass (1.1.0) + google-api-client (~> 0.13) + google-api-client (0.23.9) + addressable (~> 2.5, >= 2.5.1) + googleauth (>= 0.5, < 0.7.0) + httpclient (>= 2.8.1, < 3.0) + mime-types (~> 3.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.0) + signet (~> 0.9) + googleauth (0.6.7) + faraday (~> 0.12) + jwt (>= 1.4, < 3.0) + memoist (~> 0.16) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (~> 0.7) + gssapi (1.2.0) + ffi (>= 1.0.1) + gyoku (1.3.1) + builder (>= 2.1.2) + hashie (3.6.0) + htmlentities (4.3.4) + http-cookie (1.0.3) + domain_name (~> 0.5) + httpclient (2.8.3) + inifile (3.0.0) + inspec (3.1.3) + addressable (~> 2.4) + faraday (>= 0.9.0) + faraday_middleware (~> 0.12.2) + hashie (~> 3.4) + htmlentities + json (>= 1.8, < 3.0) + method_source (~> 0.8) + mixlib-log + multipart-post + parallel (~> 1.9) + parslet (~> 1.5) + pry (~> 0) + rspec (~> 3) + rspec-its (~> 1.2) + rubyzip (~> 1.2, >= 1.2.2) + semverse + sslshake (~> 1.2) + term-ansicolor + thor (~> 0.20) + tomlrb (~> 1.2) + train (~> 1.5, >= 1.5.11) + tty-prompt (~> 0.17) + tty-table (~> 0.10) + jmespath (1.4.0) + json (2.1.0) + jwt (2.1.0) + kitchen-google (2.0.1) + gcewinpass (~> 1.1) + google-api-client (~> 0.19) + test-kitchen + kitchen-terraform (4.1.1) + dry-types (~> 0.9) + dry-validation (~> 0.10) + inspec (~> 3.0) + mixlib-shellout (~> 2.2) + test-kitchen (~> 1.23) + little-plugger (1.1.4) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) + memoist (0.16.0) + method_source (0.9.2) + mime-types (3.2.2) + mime-types-data (~> 3.2015) + mime-types-data (3.2018.0812) + mixlib-install (3.11.5) + mixlib-shellout + mixlib-versioning + thor + mixlib-log (2.0.9) + mixlib-shellout (2.4.4) + mixlib-versioning (1.2.7) + ms_rest (0.7.3) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + timeliness (~> 0.3) + ms_rest_azure (0.11.0) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + faraday-cookie_jar (~> 0.0.6) + ms_rest (~> 0.7.2) + multi_json (1.13.1) + multipart-post (2.0.0) + necromancer (0.4.0) + net-scp (1.2.1) + net-ssh (>= 2.6.5) + net-ssh (4.2.0) + net-ssh-gateway (1.3.0) + net-ssh (>= 2.6.5) + nori (2.6.0) + os (1.0.0) + parallel (1.12.1) + parslet (1.8.2) + pastel (0.7.2) + equatable (~> 0.5.0) + tty-color (~> 0.4.0) + pry (0.12.2) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + public_suffix (3.0.3) + representable (3.0.4) + declarative (< 0.1.0) + declarative-option (< 0.2.0) + uber (< 0.2.0) + retriable (3.1.2) + rspec (3.8.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-core (3.8.0) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-its (1.2.0) + rspec-core (>= 3.0.0) + rspec-expectations (>= 3.0.0) + rspec-mocks (3.8.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-support (3.8.0) + rubyntlm (0.6.2) + rubyzip (1.2.2) + semverse (3.0.0) + signet (0.11.0) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) + sslshake (1.2.0) + strings (0.1.4) + strings-ansi (~> 0.1.0) + unicode-display_width (~> 1.4.0) + unicode_utils (~> 1.4.0) + strings-ansi (0.1.0) + term-ansicolor (1.7.0) + tins (~> 1.0) + test-kitchen (1.24.0) + mixlib-install (~> 3.6) + mixlib-shellout (>= 1.2, < 3.0) + net-scp (~> 1.1) + net-ssh (>= 2.9, < 5.0) + net-ssh-gateway (~> 1.2) + thor (~> 0.19) + winrm (~> 2.0) + winrm-elevated (~> 1.0) + winrm-fs (~> 1.1) + thor (0.20.3) + timeliness (0.3.8) + timers (4.2.0) + tins (1.20.2) + tomlrb (1.2.8) + train (1.6.3) + aws-sdk (~> 2) + azure_graph_rbac (~> 0.16) + azure_mgmt_key_vault (~> 0.17) + azure_mgmt_resources (~> 0.15) + docker-api (~> 1.26) + google-api-client (~> 0.23.9) + googleauth (~> 0.6.6) + inifile + json (>= 1.8, < 3.0) + mixlib-shellout (~> 2.0) + net-scp (~> 1.2) + net-ssh (>= 2.9, < 6.0) + winrm (~> 2.0) + winrm-fs (~> 1.0) + tty-color (0.4.3) + tty-cursor (0.6.0) + tty-prompt (0.18.0) + necromancer (~> 0.4.0) + pastel (~> 0.7.0) + timers (~> 4.0) + tty-cursor (~> 0.6.0) + tty-reader (~> 0.5.0) + tty-reader (0.5.0) + tty-cursor (~> 0.6.0) + tty-screen (~> 0.6.4) + wisper (~> 2.0.0) + tty-screen (0.6.5) + tty-table (0.10.0) + equatable (~> 0.5.0) + necromancer (~> 0.4.0) + pastel (~> 0.7.2) + strings (~> 0.1.0) + tty-screen (~> 0.6.4) + uber (0.1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + unicode-display_width (1.4.1) + unicode_utils (1.4.0) + winrm (2.3.1) + builder (>= 2.1.2) + erubis (~> 2.7) + gssapi (~> 1.2) + gyoku (~> 1.0) + httpclient (~> 2.2, >= 2.2.0.2) + logging (>= 1.6.1, < 3.0) + nori (~> 2.0) + rubyntlm (~> 0.6.0, >= 0.6.1) + winrm-elevated (1.1.1) + winrm (~> 2.0) + winrm-fs (~> 1.0) + winrm-fs (1.3.2) + erubis (~> 2.7) + logging (>= 1.6.1, < 3.0) + rubyzip (~> 1.1) + winrm (~> 2.0) + wisper (2.0.0) + +PLATFORMS + ruby + +DEPENDENCIES + inspec (~> 3.1.0)! + kitchen-google (~> 2.0)! + kitchen-terraform (~> 4.1)! + +RUBY VERSION + ruby 2.4.2p198 + +BUNDLED WITH + 1.17.3 diff --git a/modules/gcs-object/README.md b/modules/gcs-object/README.md new file mode 100644 index 0000000..3a00a5b --- /dev/null +++ b/modules/gcs-object/README.md @@ -0,0 +1,138 @@ +# terraform-gcp-get-gcs-object +# object fetching for Google Cloud GCS based secret management Terraform Module + +This is a submodule to fetch an object in a GCS bucket. + +## Example usage +See the living [barebones](./examples/barebones-test-fixture) test fixture example for usage. + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| bucket | The bucket to fetch the object from | string | - | yes | +| duration | The duration of the signed URL (defaults to 1m) | string | `1m` | no | +| path | The path to the desired object within the bucket | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| contents | The contents of the requested GCS object | + +[^]: (autogen_docs_end) + +## Requirements +### Terraform plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.10.x +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v1.10.0 + +### Permissions +In order to execute this module, you must have the roles and permissions for creating buckets. + +## Install + +### Terraform +Be sure you have the correct Terraform version (0.10.x), you can choose the binary here: +- https://releases.hashicorp.com/terraform/ + +### Terraform plugins + +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v1.10.0 + +## Development + +### File structure +The project has the following folders and files: + +- ./gcs-object: root folder +- ./examples/barebones-test-fixture: example for using this module +- ./test: Folders with files for testing the module (see Testing section on this file) +- ./main.tf: main file for this module, contains primary logic for operate the module +- ./variables.tf: all the variables for the module +- ./outputs.tf: the outputs of the module +- ./readme.MD: this file + +## Testing + +### Requirements +- [bundler](https://github.com/bundler/bundler) +- [gcloud](https://cloud.google.com/sdk/install) +- [terraform-docs](https://github.com/segmentio/terraform-docs/releases) 0.3.0 +- [docker](https://docker.com) +- [openssl](https://www.openssl.org/) - This is only used to generate a random suffix for bucket names + +### Integration test + +Integration tests are run though [test-kitchen](https://github.com/test-kitchen/test-kitchen), [kitchen-terraform](https://github.com/newcontext-oss/kitchen-terraform), and [InSpec](https://github.com/inspec/inspec). + + +The test-kitchen instances in `test/fixtures/` wrap identically-named examples in the `examples/` directory. + +#### Setup + +1. Configure the [test fixtures](#test-configuration) +2. Download a Service Account key with the necessary permissions and put it in the module's root directory with the name `service-account-credentials.json`. + Running the whole set of integration tests (from the repo root) is the recommended way of running the tests. Note that running the repo's integration tests (make test_integration) will copy the service-account-credentials.json file from the root to here. +3. To run this specific test, build and run the test in a Docker container (specify your project name in the build-arg): + ```bash + docker build . -f Dockerfile -t ubuntu-test-kitchen-terraform --build-arg RANDOM_SUFFIX=$(shell openssl rand -hex 5) --build-arg PROJECT_NAME=sample-project-name --build-arg GOOGLE_APPLICATION_CREDENTIALS=service-account-credentials.json + ``` + +4. This will build and run the kitchen tests for the module. If something fails, you might need to adjust the last line in the Dockerfile, as the image isn't fully built until that line passes. +5. Remove the kitchen call as needed, then connect to the docker tag for debugging. + ```bash + docker run -it -v $PWD:/root/live_workspace -e "PROJECT_NAME=sample-project-name" -w /root/live_workspace ubuntu-test-kitchen-terraform + ``` + + The module root directory will be loaded into the Docker container at `/root/live_workspace/`. + Run kitchen-terraform to test the infrastructure: + + 1. `kitchen create` creates Terraform state and downloads modules, if applicable. + 2. `kitchen converge` creates the underlying resources. Run `kitchen converge ` to create resources for a specific test case. + 3. `kitchen verify` tests the created infrastructure. Run `kitchen verify ` to run a specific test case. + 4. `kitchen destroy` tears down the underlying resources created by `kitchen converge`. Run `kitchen destroy ` to tear down resources for a specific test case. + +#### Test configuration + +A random suffix of characters is applied to bucket names to provide uniqueness in the tests, as GCS bucket names must be globally unique. + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Linting +The makefile in this project will lint or sometimes just format any shell, +Python, golang, Terraform, or Dockerfiles. The linters will only be run if +the makefile finds files with the appropriate file extension. + +All of the linter checks are in the default make target, so you just have to +run + +``` +make -s +``` + +The -s is for 'silent'. Successful output looks like this, though there are currently failing files. + +``` +Running shellcheck +Running flake8 +Running gofmt +Running terraform validate +Running hadolint on Dockerfiles +Test passed - Verified all file Apache 2 headers +``` + +The linters +are as follows: +* Shell - shellcheck. Can be found in homebrew +* Golang - gofmt. gofmt comes with the standard golang installation. golang +is a compiled language so there is no standard linter. +* Terraform - terraform has a built-in linter in the 'terraform validate' +command. +* Dockerfiles - hadolint. Can be found in homebrew \ No newline at end of file diff --git a/modules/gcs-object/data.tf b/modules/gcs-object/data.tf new file mode 100644 index 0000000..3c32ab7 --- /dev/null +++ b/modules/gcs-object/data.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_storage_object_signed_url" "file_url" { + bucket = "${var.bucket}" + path = "${var.path}" + duration = "${var.duration}" +} + +data "http" "remote_contents" { + url = "${data.google_storage_object_signed_url.file_url.signed_url}" + depends_on = ["null_resource.force-wait"] +} diff --git a/modules/gcs-object/examples/barebones-test-fixture/file.txt b/modules/gcs-object/examples/barebones-test-fixture/file.txt new file mode 100644 index 0000000..3b18e51 --- /dev/null +++ b/modules/gcs-object/examples/barebones-test-fixture/file.txt @@ -0,0 +1 @@ +hello world diff --git a/modules/gcs-object/examples/barebones-test-fixture/main.tf b/modules/gcs-object/examples/barebones-test-fixture/main.tf new file mode 100644 index 0000000..3d4651b --- /dev/null +++ b/modules/gcs-object/examples/barebones-test-fixture/main.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.11.7" +} + +provider "google" { + project = "${var.project_name}" + region = "${var.region}" + credentials = "${file(var.credentials_file_path)}" +} + +provider "random" {} + +resource "random_string" "suffix" { + length = 8 + special = false + upper = false +} + +locals { + bucket_name = "random-bucket-${random_string.suffix.result}" + file_object = "file.txt" +} + +module "get_foo" { + source = "../.." + bucket = "${local.bucket_name}" + path = "/foo/${local.file_object}" +} + +resource "google_storage_bucket_object" "foo" { + name = "/foo/${local.file_object}" + source = "${path.module}/${local.file_object}" + bucket = "${local.bucket_name}" + depends_on = ["google_storage_bucket.test"] +} + +resource "google_storage_bucket" "test" { + name = "${local.bucket_name}" + force_destroy = "true" +} diff --git a/modules/gcs-object/examples/barebones-test-fixture/outputs.tf b/modules/gcs-object/examples/barebones-test-fixture/outputs.tf new file mode 100644 index 0000000..7b4d3ed --- /dev/null +++ b/modules/gcs-object/examples/barebones-test-fixture/outputs.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "foo_id" { + description = "description" + value = "bar" +} + +output "project_name" { + value = "${var.project_name}" +} + +output "region" { + value = "${var.region}" +} + +output "contents" { + description = "description" + value = "${module.get_foo.contents}" +} diff --git a/modules/gcs-object/examples/barebones-test-fixture/terraform.tfvars b/modules/gcs-object/examples/barebones-test-fixture/terraform.tfvars new file mode 100644 index 0000000..ba3193c --- /dev/null +++ b/modules/gcs-object/examples/barebones-test-fixture/terraform.tfvars @@ -0,0 +1,2 @@ +project_name="simple-sample-project-cae8" +credentials_file_path="/cftk/workdir/service-account-credentials.json" \ No newline at end of file diff --git a/modules/gcs-object/examples/barebones-test-fixture/variables.tf b/modules/gcs-object/examples/barebones-test-fixture/variables.tf new file mode 100644 index 0000000..5e33b18 --- /dev/null +++ b/modules/gcs-object/examples/barebones-test-fixture/variables.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "credentials_file_path" { + description = "path to creds" +} + +variable "project_name" { + description = "name of the GCP project to create resources within" +} + +variable "environment" { + description = "the environment in which we create the test resources" + default = "dev" +} + +variable "region" { + description = "Region in which we create resources" + default = "us-west1" +} diff --git a/modules/gcs-object/main.tf b/modules/gcs-object/main.tf new file mode 100644 index 0000000..3f55229 --- /dev/null +++ b/modules/gcs-object/main.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + There is nothing here as the data.tf contains the necessary logic. +*/ + +resource "null_resource" "force-wait" { + +} \ No newline at end of file diff --git a/modules/gcs-object/outputs.tf b/modules/gcs-object/outputs.tf new file mode 100644 index 0000000..85b04eb --- /dev/null +++ b/modules/gcs-object/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "contents" { + description = "The contents of the requested GCS object" + value = "${data.http.remote_contents.body}" +} diff --git a/modules/gcs-object/test/integration/barebones/controls/service_spec.rb b/modules/gcs-object/test/integration/barebones/controls/service_spec.rb new file mode 100644 index 0000000..6aa9378 --- /dev/null +++ b/modules/gcs-object/test/integration/barebones/controls/service_spec.rb @@ -0,0 +1,27 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'rubygems' +require 'json' +# this is the prescribed way to create variables in test from terraform test outputs +contents = attribute('contents') + +# this main control block is referenced in the .kitchen.yml file. All tests fall within it. +control "get-gcs-object" do + describe 'Output validation' do + it 'contains the right content' do + expect(contents).to match "hello world" + end + end +end diff --git a/modules/gcs-object/test/integration/barebones/inspec.lock b/modules/gcs-object/test/integration/barebones/inspec.lock new file mode 100644 index 0000000..d4117b0 --- /dev/null +++ b/modules/gcs-object/test/integration/barebones/inspec.lock @@ -0,0 +1,8 @@ +--- +lockfile_version: 1 +depends: +- name: inspec-gcp + resolved_source: + url: https://github.com/inspec/inspec-gcp/archive/master.tar.gz + sha256: 63922bb632071d515627fdfeb0d437ae3dadc39388d748e6bc627a85b2cca257 + version_constraints: [] diff --git a/modules/gcs-object/test/integration/barebones/inspec.yml b/modules/gcs-object/test/integration/barebones/inspec.yml new file mode 100644 index 0000000..e73b570 --- /dev/null +++ b/modules/gcs-object/test/integration/barebones/inspec.yml @@ -0,0 +1,12 @@ +--- +name: default +depends: + - name: inspec-gcp + url: https://github.com/inspec/inspec-gcp/archive/master.tar.gz +attributes: +- name: foo_id + required: true + type: string +- name: contents + required: true + type: string \ No newline at end of file diff --git a/modules/gcs-object/variables.tf b/modules/gcs-object/variables.tf new file mode 100644 index 0000000..e370fdf --- /dev/null +++ b/modules/gcs-object/variables.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "bucket" { + description = "The bucket to fetch the object from" +} + +variable "path" { + description = "The path to the desired object within the bucket" +} + +variable "duration" { + description = "The duration of the signed URL (defaults to 1m)" + default = "1m" +} diff --git a/modules/secret-infrastructure/.kitchen.yml b/modules/secret-infrastructure/.kitchen.yml new file mode 100644 index 0000000..cf48d51 --- /dev/null +++ b/modules/secret-infrastructure/.kitchen.yml @@ -0,0 +1,19 @@ +driver: + name: terraform + +provisioner: + name: terraform + +platforms: + - name: gcp + +verifier: + name: terraform + systems: + - name: bucket + backend: local + +suites: + - name: "bucket" + driver: + root_module_directory: test/fixtures/bucket \ No newline at end of file diff --git a/modules/secret-infrastructure/Dockerfile b/modules/secret-infrastructure/Dockerfile new file mode 100644 index 0000000..f7f5360 --- /dev/null +++ b/modules/secret-infrastructure/Dockerfile @@ -0,0 +1,56 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:bionic + +RUN apt-get update -y && \ + apt-get install -y software-properties-common && \ + apt-add-repository -y ppa:rael-gc/rvm && \ + apt-get update -y && \ + apt-get install rvm -y && \ + /bin/bash -l -c "rvm install 2.4.2 && \ + echo 'gem: --no-ri --no-rdoc' > ~/.gemrc && \ + gem install bundler --no-ri --no-rdoc" + +RUN apt-get install -y unzip wget ssh git && \ + wget https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip && \ + unzip terraform_0.11.11_linux_amd64.zip && \ + mv terraform /usr/local/bin/ + +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update -y && apt-get install google-cloud-sdk -y + +COPY Gemfile ./ +RUN /bin/bash -l -c "bundle install" + +ENV WORKDIR /root/static_build +WORKDIR $WORKDIR +COPY . . +ARG GOOGLE_APPLICATION_CREDENTIALS +ENV CREDENTIALS_FILE credentials.json +COPY $GOOGLE_APPLICATION_CREDENTIALS $WORKDIR/$CREDENTIALS_FILE +ENV GOOGLE_APPLICATION_CREDENTIALS=$WORKDIR/$CREDENTIALS_FILE +ARG RANDOM_SUFFIX +ENV RANDOM_SUFFIX=$RANDOM_SUFFIX + +RUN echo "alias tf_list=\"/bin/bash -l -c 'bundle exec kitchen list'\"" >> /root/.bashrc && \ + echo "alias tf_destroy=\"/bin/bash -l -c 'bundle exec kitchen destroy ; bundle exec kitchen list'\"" >> /root/.bashrc && \ + echo "alias tf_test=\"/bin/bash -l -c 'bundle exec kitchen create && bundle exec kitchen converge && bundle exec kitchen verify ; bundle exec kitchen list'\"" >> /root/.bashrc && \ + echo "alias tf_test_and_destroy=\"/bin/bash -l -c 'bundle exec kitchen create && bundle exec kitchen test --destroy always'\"" >> /root/.bashrc + +ARG PROJECT_NAME +ENV PROJECT_NAME=$PROJECT_NAME +RUN /bin/bash -l -c 'bundle exec kitchen create && bundle exec kitchen converge ; bundle exec kitchen converge && bundle exec kitchen verify && bundle exec kitchen destroy' \ No newline at end of file diff --git a/modules/secret-infrastructure/Gemfile b/modules/secret-infrastructure/Gemfile new file mode 100644 index 0000000..9f72397 --- /dev/null +++ b/modules/secret-infrastructure/Gemfile @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +ruby '2.4.2' + +source 'https://rubygems.org/' do + gem 'inspec', '~> 3.1.0' + gem 'kitchen-terraform', '~> 4.1' + gem "kitchen-google", "~> 2.0" +end diff --git a/modules/secret-infrastructure/Gemfile.lock b/modules/secret-infrastructure/Gemfile.lock new file mode 100644 index 0000000..27bbd90 --- /dev/null +++ b/modules/secret-infrastructure/Gemfile.lock @@ -0,0 +1,300 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) + aws-sdk (2.11.197) + aws-sdk-resources (= 2.11.197) + aws-sdk-core (2.11.197) + aws-sigv4 (~> 1.0) + jmespath (~> 1.0) + aws-sdk-resources (2.11.197) + aws-sdk-core (= 2.11.197) + aws-sigv4 (1.0.3) + azure_graph_rbac (0.17.0) + ms_rest_azure (~> 0.11.0) + azure_mgmt_key_vault (0.17.2) + ms_rest_azure (~> 0.11.0) + azure_mgmt_resources (0.17.2) + ms_rest_azure (~> 0.11.0) + builder (3.2.3) + coderay (1.1.2) + concurrent-ruby (1.1.4) + declarative (0.0.10) + declarative-option (0.1.0) + diff-lcs (1.3) + docker-api (1.34.2) + excon (>= 0.47.0) + multi_json + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + dry-configurable (0.7.0) + concurrent-ruby (~> 1.0) + dry-container (0.6.0) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (0.4.7) + concurrent-ruby (~> 1.0) + dry-equalizer (0.2.1) + dry-inflector (0.1.2) + dry-logic (0.4.2) + dry-container (~> 0.2, >= 0.2.6) + dry-core (~> 0.2) + dry-equalizer (~> 0.2) + dry-types (0.13.4) + concurrent-ruby (~> 1.0) + dry-container (~> 0.3) + dry-core (~> 0.4, >= 0.4.4) + dry-equalizer (~> 0.2) + dry-inflector (~> 0.1, >= 0.1.2) + dry-logic (~> 0.4, >= 0.4.2) + dry-validation (0.12.2) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (~> 0.2, >= 0.2.1) + dry-equalizer (~> 0.2) + dry-logic (~> 0.4, >= 0.4.0) + dry-types (~> 0.13.1) + equatable (0.5.0) + erubis (2.7.0) + excon (0.62.0) + faraday (0.15.4) + multipart-post (>= 1.2, < 3) + faraday-cookie_jar (0.0.6) + faraday (>= 0.7.4) + http-cookie (~> 1.0.0) + faraday_middleware (0.12.2) + faraday (>= 0.7.4, < 1.0) + ffi (1.9.25) + gcewinpass (1.1.0) + google-api-client (~> 0.13) + google-api-client (0.23.9) + addressable (~> 2.5, >= 2.5.1) + googleauth (>= 0.5, < 0.7.0) + httpclient (>= 2.8.1, < 3.0) + mime-types (~> 3.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.0) + signet (~> 0.9) + googleauth (0.6.7) + faraday (~> 0.12) + jwt (>= 1.4, < 3.0) + memoist (~> 0.16) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (~> 0.7) + gssapi (1.2.0) + ffi (>= 1.0.1) + gyoku (1.3.1) + builder (>= 2.1.2) + hashie (3.6.0) + htmlentities (4.3.4) + http-cookie (1.0.3) + domain_name (~> 0.5) + httpclient (2.8.3) + inifile (3.0.0) + inspec (3.1.3) + addressable (~> 2.4) + faraday (>= 0.9.0) + faraday_middleware (~> 0.12.2) + hashie (~> 3.4) + htmlentities + json (>= 1.8, < 3.0) + method_source (~> 0.8) + mixlib-log + multipart-post + parallel (~> 1.9) + parslet (~> 1.5) + pry (~> 0) + rspec (~> 3) + rspec-its (~> 1.2) + rubyzip (~> 1.2, >= 1.2.2) + semverse + sslshake (~> 1.2) + term-ansicolor + thor (~> 0.20) + tomlrb (~> 1.2) + train (~> 1.5, >= 1.5.11) + tty-prompt (~> 0.17) + tty-table (~> 0.10) + jmespath (1.4.0) + json (2.1.0) + jwt (2.1.0) + kitchen-google (2.0.1) + gcewinpass (~> 1.1) + google-api-client (~> 0.19) + test-kitchen + kitchen-terraform (4.1.1) + dry-types (~> 0.9) + dry-validation (~> 0.10) + inspec (~> 3.0) + mixlib-shellout (~> 2.2) + test-kitchen (~> 1.23) + little-plugger (1.1.4) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) + memoist (0.16.0) + method_source (0.9.2) + mime-types (3.2.2) + mime-types-data (~> 3.2015) + mime-types-data (3.2018.0812) + mixlib-install (3.11.5) + mixlib-shellout + mixlib-versioning + thor + mixlib-log (2.0.9) + mixlib-shellout (2.4.4) + mixlib-versioning (1.2.7) + ms_rest (0.7.3) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + timeliness (~> 0.3) + ms_rest_azure (0.11.0) + concurrent-ruby (~> 1.0) + faraday (~> 0.9) + faraday-cookie_jar (~> 0.0.6) + ms_rest (~> 0.7.2) + multi_json (1.13.1) + multipart-post (2.0.0) + necromancer (0.4.0) + net-scp (1.2.1) + net-ssh (>= 2.6.5) + net-ssh (4.2.0) + net-ssh-gateway (1.3.0) + net-ssh (>= 2.6.5) + nori (2.6.0) + os (1.0.0) + parallel (1.12.1) + parslet (1.8.2) + pastel (0.7.2) + equatable (~> 0.5.0) + tty-color (~> 0.4.0) + pry (0.12.2) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + public_suffix (3.0.3) + representable (3.0.4) + declarative (< 0.1.0) + declarative-option (< 0.2.0) + uber (< 0.2.0) + retriable (3.1.2) + rspec (3.8.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-core (3.8.0) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-its (1.2.0) + rspec-core (>= 3.0.0) + rspec-expectations (>= 3.0.0) + rspec-mocks (3.8.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-support (3.8.0) + rubyntlm (0.6.2) + rubyzip (1.2.2) + semverse (3.0.0) + signet (0.11.0) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) + sslshake (1.2.0) + strings (0.1.4) + strings-ansi (~> 0.1.0) + unicode-display_width (~> 1.4.0) + unicode_utils (~> 1.4.0) + strings-ansi (0.1.0) + term-ansicolor (1.7.0) + tins (~> 1.0) + test-kitchen (1.24.0) + mixlib-install (~> 3.6) + mixlib-shellout (>= 1.2, < 3.0) + net-scp (~> 1.1) + net-ssh (>= 2.9, < 5.0) + net-ssh-gateway (~> 1.2) + thor (~> 0.19) + winrm (~> 2.0) + winrm-elevated (~> 1.0) + winrm-fs (~> 1.1) + thor (0.20.3) + timeliness (0.3.8) + timers (4.2.0) + tins (1.20.2) + tomlrb (1.2.8) + train (1.6.3) + aws-sdk (~> 2) + azure_graph_rbac (~> 0.16) + azure_mgmt_key_vault (~> 0.17) + azure_mgmt_resources (~> 0.15) + docker-api (~> 1.26) + google-api-client (~> 0.23.9) + googleauth (~> 0.6.6) + inifile + json (>= 1.8, < 3.0) + mixlib-shellout (~> 2.0) + net-scp (~> 1.2) + net-ssh (>= 2.9, < 6.0) + winrm (~> 2.0) + winrm-fs (~> 1.0) + tty-color (0.4.3) + tty-cursor (0.6.0) + tty-prompt (0.18.0) + necromancer (~> 0.4.0) + pastel (~> 0.7.0) + timers (~> 4.0) + tty-cursor (~> 0.6.0) + tty-reader (~> 0.5.0) + tty-reader (0.5.0) + tty-cursor (~> 0.6.0) + tty-screen (~> 0.6.4) + wisper (~> 2.0.0) + tty-screen (0.6.5) + tty-table (0.10.0) + equatable (~> 0.5.0) + necromancer (~> 0.4.0) + pastel (~> 0.7.2) + strings (~> 0.1.0) + tty-screen (~> 0.6.4) + uber (0.1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + unicode-display_width (1.4.1) + unicode_utils (1.4.0) + winrm (2.3.1) + builder (>= 2.1.2) + erubis (~> 2.7) + gssapi (~> 1.2) + gyoku (~> 1.0) + httpclient (~> 2.2, >= 2.2.0.2) + logging (>= 1.6.1, < 3.0) + nori (~> 2.0) + rubyntlm (~> 0.6.0, >= 0.6.1) + winrm-elevated (1.1.1) + winrm (~> 2.0) + winrm-fs (~> 1.0) + winrm-fs (1.3.2) + erubis (~> 2.7) + logging (>= 1.6.1, < 3.0) + rubyzip (~> 1.1) + winrm (~> 2.0) + wisper (2.0.0) + +PLATFORMS + ruby + +DEPENDENCIES + inspec (~> 3.1.0)! + kitchen-google (~> 2.0)! + kitchen-terraform (~> 4.1)! + +RUBY VERSION + ruby 2.4.2p198 + +BUNDLED WITH + 1.17.3 diff --git a/modules/secret-infrastructure/README.md b/modules/secret-infrastructure/README.md new file mode 100644 index 0000000..b6ac212 --- /dev/null +++ b/modules/secret-infrastructure/README.md @@ -0,0 +1,169 @@ +# terraform-google-secret/secret-infrastructure + +# Bucket infrastructure for Google Cloud GCS based secret management Terraform Module + +This Terraform submodule handles the creation of buckets for storing app and shared secrets. + + +## Usage +An example is included under the root examples folder at [examples/bucket_example](../examples/bucket_example). + +### Creating the buckets + +A simple example to create buckets is as follows: + +```hcl +module "secret-storage" { + source = "./secret-infrastructure" + project_name = "your-secret-storage-project" + application_list = ["webapp", "service1"] + env_list = ["dev", "qa", "production"] + credentials_file_path = "service-account-creds.json" +} +``` +This will create buckets with of the form: appname-env-secrets +Using the above as an example, this will create: +* webapp-dev-secrets +* webapp-qa-secrets +* webapp-prod-secrets +* service1-dev-secrets +* service1-qa-secrets +* service1-prod-secrets + +Along with the shared buckets per environment +* shared-projectname-dev-secrets +* shared-projectname-qa-secrets +* shared-projectname-prod-secrets + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| application\_list | The list of application names that will store secrets | list | `` | no | +| credentials\_file\_path | GCP credentials fils | string | - | yes | +| env\_list | The list of environments for secrets | list | `` | no | +| project\_name | The name of the project this applies to | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| app-buckets | - | +| shared-buckets | - | + +[^]: (autogen_docs_end) + +## Requirements +### Terraform plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.10.x +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v1.10.0 + +### Permissions +In order to execute this module, you must have the roles and permissions for creating buckets. + +## Install + +### Terraform +Be sure you have the correct Terraform version (0.10.x), you can choose the binary here: +- https://releases.hashicorp.com/terraform/ + +### Terraform plugins + +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v1.10.0 + +## Development + +### File structure +The project has the following folders and files: + +- ./secret-infrastructure: root folder +- ../examples/bucket_example: example for using this module +- ./test: Folders with files for testing the module (see Testing section on this file) +- ./main.tf: main file for this module, contains primary logic for operate the module +- ./variables.tf: all the variables for the module +- ./outputs.tf: the outputs of the module +- ./readme.MD: this file + +## Testing + +### Requirements +- [bundler](https://github.com/bundler/bundler) +- [gcloud](https://cloud.google.com/sdk/install) +- [terraform-docs](https://github.com/segmentio/terraform-docs/releases) 0.3.0 +- [docker](https://docker.com) +- [openssl](https://www.openssl.org/) - This is only used to generate a random suffix for bucket names + +### Integration test + +Integration tests are run though [test-kitchen](https://github.com/test-kitchen/test-kitchen), [kitchen-terraform](https://github.com/newcontext-oss/kitchen-terraform), and [InSpec](https://github.com/inspec/inspec). + + +The test-kitchen instances in `test/fixtures/` wrap identically-named examples in the `examples/` directory. + +#### Setup + +1. Configure the [test fixtures](#test-configuration) +2. Download a Service Account key with the necessary permissions and put it in the module's root directory with the name `service-account-credentials.json`. + Running the whole set of integration tests (from the repo root) is the recommended way of running the tests. Note that running the repo's integration tests (make test_integration) will copy the service-account-credentials.json file from the root to here. +3. To run this specific test, build and run the test in a Docker container (specify your project name in the build-arg): + ```bash + docker build . -f Dockerfile -t ubuntu-test-kitchen-terraform --build-arg RANDOM_SUFFIX=$(shell openssl rand -hex 5) --build-arg PROJECT_NAME=sample-project-name --build-arg GOOGLE_APPLICATION_CREDENTIALS=service-account-credentials.json + ``` + +4. This will build and run the kitchen tests for the module. If something fails, you might need to adjust the last line in the Dockerfile, as the image isn't fully built until that line passes. +5. Remove the kitchen call as needed, then connect to the docker tag for debugging. + ```bash + docker run -it -v $PWD:/root/live_workspace -e "PROJECT_NAME=sample-project-name" -w /root/live_workspace ubuntu-test-kitchen-terraform + ``` + + The module root directory will be loaded into the Docker container at `/root/live_workspace/`. + Run kitchen-terraform to test the infrastructure: + + 1. `kitchen create` creates Terraform state and downloads modules, if applicable. + 2. `kitchen converge` creates the underlying resources. Run `kitchen converge ` to create resources for a specific test case. + 3. `kitchen verify` tests the created infrastructure. Run `kitchen verify ` to run a specific test case. + 4. `kitchen destroy` tears down the underlying resources created by `kitchen converge`. Run `kitchen destroy ` to tear down resources for a specific test case. + +#### Test configuration + +A random suffix of characters is applied to bucket names to provide uniqueness in the tests, as GCS bucket names must be globally unique. + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Linting +The makefile in this project will lint or sometimes just format any shell, +Python, golang, Terraform, or Dockerfiles. The linters will only be run if +the makefile finds files with the appropriate file extension. + +All of the linter checks are in the default make target, so you just have to +run + +``` +make -s +``` + +The -s is for 'silent'. Successful output looks like this, though there are currently failing files. + +``` +Running shellcheck +Running flake8 +Running gofmt +Running terraform validate +Running hadolint on Dockerfiles +Test passed - Verified all file Apache 2 headers +``` + +The linters +are as follows: +* Shell - shellcheck. Can be found in homebrew +* Golang - gofmt. gofmt comes with the standard golang installation. golang +is a compiled language so there is no standard linter. +* Terraform - terraform has a built-in linter in the 'terraform validate' +command. +* Dockerfiles - hadolint. Can be found in homebrew \ No newline at end of file diff --git a/modules/secret-infrastructure/infra/README.md b/modules/secret-infrastructure/infra/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/secret-infrastructure/main.tf b/modules/secret-infrastructure/main.tf new file mode 100644 index 0000000..5fb554c --- /dev/null +++ b/modules/secret-infrastructure/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals {} + +/****************************************** + Provider configuration + *****************************************/ +provider "google" { + credentials = "${file(var.credentials_file_path)}" +} + +resource "google_storage_bucket" "secrets" { + count = "${length(var.env_list)}" + name = "shared-${var.project_name}-${element(var.env_list, count.index)}-secrets" + storage_class = "MULTI_REGIONAL" + project = "${var.project_name}" + + versioning { + enabled = true + } + + force_destroy = true +} + +resource "google_storage_bucket" "app-secrets" { + count = "${length(var.application_list) * length(var.env_list)}" + name = "${element(var.application_list, count.index)}-${element(var.env_list, (count.index % length(var.env_list)))}-secrets" + storage_class = "MULTI_REGIONAL" + project = "${var.project_name}" + + versioning { + enabled = true + } + + force_destroy = true +} diff --git a/modules/secret-infrastructure/outputs.tf b/modules/secret-infrastructure/outputs.tf new file mode 100644 index 0000000..052d777 --- /dev/null +++ b/modules/secret-infrastructure/outputs.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "app-buckets" { + value = ["${google_storage_bucket.app-secrets.*.name}"] +} + +output "shared-buckets" { + value = ["${google_storage_bucket.secrets.*.name}"] +} diff --git a/modules/secret-infrastructure/test/fixtures/bucket/README.md b/modules/secret-infrastructure/test/fixtures/bucket/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/secret-infrastructure/test/fixtures/bucket/main.tf b/modules/secret-infrastructure/test/fixtures/bucket/main.tf new file mode 100644 index 0000000..06234a4 --- /dev/null +++ b/modules/secret-infrastructure/test/fixtures/bucket/main.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + app_list = ["app1${var.random_suffix}", "app2${var.random_suffix}"] + env_list = ["dev", "qa", "prod"] +} + +module "create-buckets" { + source = "../../../" + project_name = "${var.project_name}" + application_list = "${local.app_list}" + env_list = "${local.env_list}" + credentials_file_path = "${var.credentials_file_path}" +} diff --git a/modules/secret-infrastructure/test/fixtures/bucket/outputs.tf b/modules/secret-infrastructure/test/fixtures/bucket/outputs.tf new file mode 100644 index 0000000..d819fa4 --- /dev/null +++ b/modules/secret-infrastructure/test/fixtures/bucket/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "app-buckets" { + value = ["${module.create-buckets.app-buckets}"] +} + +output "shared-buckets" { + value = ["${module.create-buckets.shared-buckets}"] +} + +output "app-list" { + value = "${local.app_list}" +} + +output "env-list" { + value = "${local.env_list}" +} + +output "project-name" { + value = "${var.project_name}" +} diff --git a/modules/secret-infrastructure/test/fixtures/bucket/terraform.tfvars b/modules/secret-infrastructure/test/fixtures/bucket/terraform.tfvars new file mode 100644 index 0000000..6bf58e3 --- /dev/null +++ b/modules/secret-infrastructure/test/fixtures/bucket/terraform.tfvars @@ -0,0 +1,3 @@ +project_name="simple-sample-project-cae8" +credentials_file_path="/cftk/workdir/service-account-credentials.json" +random_suffix="acayb" \ No newline at end of file diff --git a/modules/secret-infrastructure/test/fixtures/bucket/variables.tf b/modules/secret-infrastructure/test/fixtures/bucket/variables.tf new file mode 100644 index 0000000..c4776f5 --- /dev/null +++ b/modules/secret-infrastructure/test/fixtures/bucket/variables.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_name" { + description = "Name of test project" +} + +variable "credentials_file_path" { + description = "Credentials for access" +} + +variable "random_suffix" { + description = "random characters appended to bucket names for uniqueness in tests" +} diff --git a/modules/secret-infrastructure/test/integration/bucket/controls/inspec.lock b/modules/secret-infrastructure/test/integration/bucket/controls/inspec.lock new file mode 100644 index 0000000..e687b9b --- /dev/null +++ b/modules/secret-infrastructure/test/integration/bucket/controls/inspec.lock @@ -0,0 +1,3 @@ +--- +lockfile_version: 1 +depends: [] diff --git a/modules/secret-infrastructure/test/integration/bucket/controls/service_spec.rb b/modules/secret-infrastructure/test/integration/bucket/controls/service_spec.rb new file mode 100644 index 0000000..b51b187 --- /dev/null +++ b/modules/secret-infrastructure/test/integration/bucket/controls/service_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'rubygems' +require 'json' + +# this is the prescribed way to create variables in test from terraform test outputs +app_buckets = attribute('app-buckets') +shared_buckets = attribute('shared-buckets') +expected_app_list = attribute('app-list') +expected_env_list = attribute('env-list') +project_name = attribute('project-name') + +control "check app buckets" do + describe 'App buckets' do + it 'has the right amount of buckets' do + expect(app_buckets.length).to match 6 + end + it 'has the right names for the buckets' do + expected_app_list.each do |app| + expected_env_list.each do |env| + expect(app_buckets).to include("#{app}-#{env}-secrets") + end + end + end + end +end + +control "check shared buckets" do + describe 'Shared buckets' do + it 'has the right amount of buckets' do + expect(shared_buckets.length).to match 3 + end + it 'has the right names for the buckets' do + expected_env_list.each do |env| + expect(shared_buckets).to include("shared-#{project_name}-#{env}-secrets") + end + end + end +end + diff --git a/modules/secret-infrastructure/test/integration/bucket/inspec.lock b/modules/secret-infrastructure/test/integration/bucket/inspec.lock new file mode 100644 index 0000000..e687b9b --- /dev/null +++ b/modules/secret-infrastructure/test/integration/bucket/inspec.lock @@ -0,0 +1,3 @@ +--- +lockfile_version: 1 +depends: [] diff --git a/modules/secret-infrastructure/test/integration/bucket/inspec.yml b/modules/secret-infrastructure/test/integration/bucket/inspec.yml new file mode 100644 index 0000000..6af08db --- /dev/null +++ b/modules/secret-infrastructure/test/integration/bucket/inspec.yml @@ -0,0 +1,19 @@ +--- +name: bucket +version: 0.1 +attributes: +- name: app-buckets + required: true + type: array +- name: shared-buckets + required: true + type: array +- name: app-list + required: true + type: array +- name: env-list + required: true + type: array +- name: project-name + required: true + type: string diff --git a/modules/secret-infrastructure/variables.tf b/modules/secret-infrastructure/variables.tf new file mode 100644 index 0000000..fb336d2 --- /dev/null +++ b/modules/secret-infrastructure/variables.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "application_list" { + description = "The list of application names that will store secrets" + type = "list" + default = [] +} + +variable "env_list" { + description = "The list of environments for secrets" + type = "list" + default = [] +} + +variable "project_name" { + description = "The name of the project this applies to" +} + +variable "credentials_file_path" { + description = "GCP credentials fils" +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..1bb9114 --- /dev/null +++ b/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "contents" { + description = "The actual value of the requested secret" + value = "${module.secret.contents}" +} diff --git a/test/boilerplate/boilerplate.Dockerfile.txt b/test/boilerplate/boilerplate.Dockerfile.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.Dockerfile.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.Makefile.txt b/test/boilerplate/boilerplate.Makefile.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.Makefile.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.go.txt b/test/boilerplate/boilerplate.go.txt new file mode 100644 index 0000000..557e16f --- /dev/null +++ b/test/boilerplate/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/test/boilerplate/boilerplate.py.txt b/test/boilerplate/boilerplate.py.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.py.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.rb.txt b/test/boilerplate/boilerplate.rb.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.rb.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.sh.txt b/test/boilerplate/boilerplate.sh.txt new file mode 100644 index 0000000..2e94f3e --- /dev/null +++ b/test/boilerplate/boilerplate.sh.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.tf.txt b/test/boilerplate/boilerplate.tf.txt new file mode 100644 index 0000000..cfccff8 --- /dev/null +++ b/test/boilerplate/boilerplate.tf.txt @@ -0,0 +1,15 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/test/boilerplate/boilerplate.xml.txt b/test/boilerplate/boilerplate.xml.txt new file mode 100644 index 0000000..3d98cdc --- /dev/null +++ b/test/boilerplate/boilerplate.xml.txt @@ -0,0 +1,15 @@ + diff --git a/test/boilerplate/boilerplate.yaml.txt b/test/boilerplate/boilerplate.yaml.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.yaml.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/fixtures/fetch-secret/README.md b/test/fixtures/fetch-secret/README.md new file mode 100644 index 0000000..e69de29 diff --git a/test/fixtures/fetch-secret/file.txt b/test/fixtures/fetch-secret/file.txt new file mode 100644 index 0000000..1fc6584 --- /dev/null +++ b/test/fixtures/fetch-secret/file.txt @@ -0,0 +1 @@ +Test object diff --git a/test/fixtures/fetch-secret/main.tf b/test/fixtures/fetch-secret/main.tf new file mode 100644 index 0000000..f14e12d --- /dev/null +++ b/test/fixtures/fetch-secret/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + app_name = "app1${var.random_suffix}" + app_list = ["app1${var.random_suffix}", "app2${var.random_suffix}"] + env_list = ["dev", "qa", "prod"] + file_path = "${path.module}/file.txt" + file_content = "${file(local.file_path)}" +} + +provider "google" { + credentials = "${file(var.credentials_file_path)}" +} + + +resource "google_storage_bucket_object" "test-object" { + name = "test-secret.txt" + content = "${local.file_content}" + bucket = "${element(module.create-buckets.app-buckets,0)}" +} + +module "create-buckets" { + source = "../../../modules/secret-infrastructure" + project_name = "${var.project_name}" + credentials_file_path = "${var.credentials_file_path}" + application_list = "${local.app_list}" + env_list = "${local.env_list}" +} + +module "fetch-secret-test" { + source = "../../../" + credentials_file_path = "${var.credentials_file_path}" + application_name = "${local.app_name}" + secret = "test-secret" + env = "dev" +} diff --git a/test/fixtures/fetch-secret/output.tf b/test/fixtures/fetch-secret/output.tf new file mode 100644 index 0000000..b04eeb9 --- /dev/null +++ b/test/fixtures/fetch-secret/output.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//output "app-buckets" { +// value = ["${module.create-buckets.app-buckets}"] +//} +// +//output "shared-buckets" { +// value = ["${module.create-buckets.shared-buckets}"] +//} + +output "app-list" { + value = "${local.app_list}" +} + +output "env-list" { + value = "${local.env_list}" +} + +output "project-name" { + value = "${var.project_name}" +} + +output "mysecret" { + value = "${module.fetch-secret-test.contents}" + sensitive = false +} diff --git a/test/fixtures/fetch-secret/terraform.tfvars b/test/fixtures/fetch-secret/terraform.tfvars new file mode 100644 index 0000000..13c220e --- /dev/null +++ b/test/fixtures/fetch-secret/terraform.tfvars @@ -0,0 +1,3 @@ +project_name="simple-sample-project-cae8" +credentials_file_path="/cftk/workdir/service-account-credentials.json" +random_suffix="acbey" \ No newline at end of file diff --git a/test/fixtures/fetch-secret/variables.tf b/test/fixtures/fetch-secret/variables.tf new file mode 100644 index 0000000..c4776f5 --- /dev/null +++ b/test/fixtures/fetch-secret/variables.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_name" { + description = "Name of test project" +} + +variable "credentials_file_path" { + description = "Credentials for access" +} + +variable "random_suffix" { + description = "random characters appended to bucket names for uniqueness in tests" +} diff --git a/test/integration/fetch-secret/controls/inspec.lock b/test/integration/fetch-secret/controls/inspec.lock new file mode 100644 index 0000000..e687b9b --- /dev/null +++ b/test/integration/fetch-secret/controls/inspec.lock @@ -0,0 +1,3 @@ +--- +lockfile_version: 1 +depends: [] diff --git a/test/integration/fetch-secret/controls/service_spec.rb b/test/integration/fetch-secret/controls/service_spec.rb new file mode 100644 index 0000000..6104a7c --- /dev/null +++ b/test/integration/fetch-secret/controls/service_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'rubygems' +require 'json' + +# this is the prescribed way to create variables in test from terraform test outputs +app_buckets = attribute('app-buckets') +shared_buckets = attribute('shared-buckets') +expected_app_list = attribute('app-list') +expected_env_list = attribute('env-list') +project_name = attribute('project-name') +secret = attribute('mysecret') + +control "check app buckets" do + describe 'App buckets' do + it 'has the right amount of buckets' do + expect(app_buckets.length).to match 6 + end + it 'has the right names for the buckets' do + expected_app_list.each do |app| + expected_env_list.each do |env| + expect(app_buckets).to include("#{app}-#{env}-secrets") + end + end + end + end +end + +control "check shared buckets" do + describe 'Shared buckets' do + it 'has the right amount of buckets' do + expect(shared_buckets.length).to match 3 + end + it 'has the right names for the buckets' do + expected_env_list.each do |env| + expect(shared_buckets).to include("shared-#{project_name}-#{env}-secrets") + end + end + end +end + +control "check secret" do + describe 'Verify secret from gcs' do + it 'has the correct value' do + expect(secret).to match "Test object\n" + end + end +end + diff --git a/test/integration/fetch-secret/inspec.lock b/test/integration/fetch-secret/inspec.lock new file mode 100644 index 0000000..e687b9b --- /dev/null +++ b/test/integration/fetch-secret/inspec.lock @@ -0,0 +1,3 @@ +--- +lockfile_version: 1 +depends: [] diff --git a/test/integration/fetch-secret/inspec.yml b/test/integration/fetch-secret/inspec.yml new file mode 100644 index 0000000..3e4106e --- /dev/null +++ b/test/integration/fetch-secret/inspec.yml @@ -0,0 +1,22 @@ +--- +name: bucket +version: 0.1 +attributes: +- name: app-buckets + required: true + type: array +- name: shared-buckets + required: true + type: array +- name: app-list + required: true + type: array +- name: env-list + required: true + type: array +- name: project-name + required: true + type: string +- name: mysecret + required: true + type: string \ No newline at end of file diff --git a/test/make.sh b/test/make.sh new file mode 100755 index 0000000..d8f6272 --- /dev/null +++ b/test/make.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This function checks to make sure that every +# shebang has a '- e' flag, which causes it +# to exit on error +function check_bash() { +find . -name "*.sh" -not -path "*/.terraform/*" | while IFS= read -d '' -r file; +do + if [[ "$file" != *"bash -e"* ]]; + then + echo "$file is missing shebang with -e"; + exit 1; + fi; +done; +} + +# This function makes sure that the required files for +# releasing to OSS are present +function basefiles() { + echo "Checking for required files" + test -f LICENSE || echo "Missing LICENSE" + test -f README.md || echo "Missing README.md" +} + +# This function runs the hadolint linter on +# every file named 'Dockerfile' +function docker() { + echo "Running hadolint on Dockerfiles" + find . -name "Dockerfile" -not -path "*/.terraform/*" -exec hadolint {} \; +} + +# This function runs 'terraform validate' against all +# files ending in '.tf' +function check_terraform() { + echo "Running terraform validate" + #shellcheck disable=SC2156 + find . -name "*.tf" -not -path "*/.terraform/*" -exec bash -c 'terraform validate --check-variables=false $(dirname "{}")' \; +} + +# This function runs 'go fmt' and 'go vet' on eery file +# that ends in '.go' +function golang() { + echo "Running go fmt and go vet" + find . -name "*.go" -not -path "*/.terraform/*" -exec go fmt {} \; + find . -name "*.go" -not -path "*/.terraform/*" -exec go vet {} \; +} + +# This function runs the flake8 linter on every file +# ending in '.py' +function check_python() { + echo "Running flake8" + find . -name "*.py" -not -path "*/.terraform/*" -exec flake8 {} \; +} + +# This function runs the shellcheck linter on every +# file ending in '.sh' +function check_shell() { + echo "Running shellcheck" + find . -name "*.sh" -not -path "*/.terraform/*" -exec shellcheck -x {} \; +} + +# This function makes sure that there is no trailing whitespace +# in any files in the project. +# There are some exclusions +function check_trailing_whitespace() { + echo "The following lines have trailing whitespace" + grep -r '[[:blank:]]$' --exclude-dir=".terraform" --exclude="*.png" --exclude="*.pyc" --exclude-dir=".git" . + rc=$? + if [ $rc = 0 ]; then + exit 1 + fi +} + +function generate_docs() { + echo "Generating markdown docs with terraform-docs" + TMPFILE=$(mktemp) + for j in `for i in $(find . -type f | grep \.tf$) ; do dirname "$i" ; done | sort -u` ; do + terraform-docs markdown "$j" > "$TMPFILE" + python helpers/combine_docfiles.py "$j/README.md" "$TMPFILE" + done + rm -f "$TMPFILE" +} diff --git a/test/test_verify_boilerplate.py b/test/test_verify_boilerplate.py new file mode 100755 index 0000000..22a3cca --- /dev/null +++ b/test/test_verify_boilerplate.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' A simple test for the verify_boilerplate python script. +This will create a set of test files, both valid and invalid, +and confirm that the has_valid_header call returns the correct +value. + +It also checks the number of files that are found by the +get_files call. +''' +from copy import deepcopy +from tempfile import mkdtemp +from shutil import rmtree +import unittest +from verify_boilerplate import has_valid_header, get_refs, get_regexs, \ + get_args, get_files + + +class AllTestCase(unittest.TestCase): + """ + All of the setup, teardown, and tests are contained in this + class. + """ + + def write_file(self, filename, content, expected): + """ + A utility method that creates test files, and adds them to + the cases that will be tested. + + Args: + filename: (string) the file name (path) to be created. + content: (list of strings) the contents of the file. + expected: (boolean) True if the header is expected to be valid, + false if not. + """ + + file = open(filename, 'w+') + for line in content: + file.write(line + "\n") + file.close() + self.cases[filename] = expected + + def create_test_files(self, tmp_path, extension, header): + """ + Creates 2 test files for .tf, .xml, .go, etc and one for + Dockerfile, and Makefile. + + The reason for the difference is that Makefile and Dockerfile + don't have an extension. These would be substantially more + difficult to create negative test cases, unless the files + were written, deleted, and re-written. + + Args: + tmp_path: (string) the path in which to create the files + extension: (string) the file extension + header: (list of strings) the header/boilerplate content + """ + + content = "\n...blah \ncould be code or could be garbage\n" + special_cases = ["Dockerfile", "Makefile"] + header_template = deepcopy(header) + valid_filename = tmp_path + extension + valid_content = header_template.append(content) + if extension not in special_cases: + # Invalid test cases for non-*file files (.tf|.py|.sh|.yaml|.xml..) + invalid_header = [] + for line in header_template: + if "2018" in line: + invalid_header.append(line.replace('2018', 'YEAR')) + else: + invalid_header.append(line) + invalid_header.append(content) + invalid_content = invalid_header + invalid_filename = tmp_path + "invalid." + extension + self.write_file(invalid_filename, invalid_content, False) + valid_filename = tmp_path + "testfile." + extension + + valid_content = header_template + self.write_file(valid_filename, valid_content, True) + + def setUp(self): + """ + Set initial counts and values, and initializes the setup of the + test files. + """ + self.cases = {} + self.tmp_path = mkdtemp() + "/" + self.my_args = get_args() + self.my_refs = get_refs(self.my_args) + self.my_regex = get_regexs() + self.prexisting_file_count = len( + get_files(self.my_refs.keys(), self.my_args)) + for key in self.my_refs: + self.create_test_files(self.tmp_path, key, + self.my_refs.get(key)) + + def tearDown(self): + """ Delete the test directory. """ + rmtree(self.tmp_path) + + def test_files_headers(self): + """ + Confirms that the expected output of has_valid_header is correct. + """ + for case in self.cases: + if self.cases[case]: + self.assertTrue(has_valid_header(case, self.my_refs, + self.my_regex)) + else: + self.assertFalse(has_valid_header(case, self.my_refs, + self.my_regex)) + + def test_invalid_count(self): + """ + Test that the initial files found isn't zero, indicating + a problem with the code. + """ + self.assertFalse(self.prexisting_file_count == 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/verify_boilerplate.py b/test/verify_boilerplate.py new file mode 100755 index 0000000..24c5cfb --- /dev/null +++ b/test/verify_boilerplate.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Verifies that all source files contain the necessary copyright boilerplate +# snippet. +# This is based on existing work +# https://github.com/kubernetes/test-infra/blob/master/hack +# /verify_boilerplate.py +from __future__ import print_function +import argparse +import glob +import os +import re +import sys + + +def get_args(): + """Parses command line arguments. + + Configures and runs argparse.ArgumentParser to extract command line + arguments. + + Returns: + An argparse.Namespace containing the arguments parsed from the + command line + """ + parser = argparse.ArgumentParser() + parser.add_argument("filenames", + help="list of files to check, " + "all files if unspecified", + nargs='*') + rootdir = os.path.dirname(__file__) + "/../" + rootdir = os.path.abspath(rootdir) + parser.add_argument( + "--rootdir", + default=rootdir, + help="root directory to examine") + + default_boilerplate_dir = os.path.join(rootdir, "test/boilerplate") + parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir) + return parser.parse_args() + + +def get_refs(ARGS): + """Converts the directory of boilerplate files into a map keyed by file + extension. + + Reads each boilerplate file's contents into an array, then adds that array + to a map keyed by the file extension. + + Returns: + A map of boilerplate lines, keyed by file extension. For example, + boilerplate.py.txt would result in the k,v pair {".py": py_lines} where + py_lines is an array containing each line of the file. + """ + refs = {} + + # Find and iterate over the absolute path for each boilerplate template + for path in glob.glob(os.path.join( + ARGS.boilerplate_dir, + "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + return refs + + +# pylint: disable=too-many-locals +def has_valid_header(filename, refs, regexs): + """Test whether a file has the correct boilerplate header. + + Tests each file against the boilerplate stored in refs for that file type + (based on extension), or by the entire filename (eg Dockerfile, Makefile). + Some heuristics are applied to remove build tags and shebangs, but little + variance in header formatting is tolerated. + + Args: + filename: A string containing the name of the file to test + refs: A map of boilerplate headers, keyed by file extension + regexs: a map of compiled regex objects used in verifying boilerplate + + Returns: + True if the file has the correct boilerplate header, otherwise returns + False. + """ + try: + with open(filename, 'r') as fp: # pylint: disable=invalid-name + data = fp.read() + except IOError: + return False + basename = os.path.basename(filename) + extension = get_file_extension(filename) + if extension: + ref = refs[extension] + else: + ref = refs[basename] + # remove build tags from the top of Go files + if extension == "go": + con = regexs["go_build_constraints"] + (data, found) = con.subn("", data, 1) + # remove shebang + elif extension == "sh" or extension == "py": + she = regexs["shebang"] + (data, found) = she.subn("", data, 1) + data = data.splitlines() + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + return False + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + year = regexs["year"] + for datum in data: + if year.search(datum): + return False + + # if we don't match the reference at this point, fail + if ref != data: + return False + return True + + +def get_file_extension(filename): + """Extracts the extension part of a filename. + + Identifies the extension as everything after the last period in filename. + + Args: + filename: string containing the filename + + Returns: + A string containing the extension in lowercase + """ + return os.path.splitext(filename)[1].split(".")[-1].lower() + + +# These directories will be omitted from header checks +SKIPPED_DIRS = [ + 'Godeps', 'third_party', '_gopath', '_output', + '.git', 'vendor', '__init__.py', 'node_modules', '.terraform', '.idea' +] + + +def normalize_files(files): + """Extracts the files that require boilerplate checking from the files + argument. + + A new list will be built. Each path from the original files argument will + be added unless it is within one of SKIPPED_DIRS. All relative paths will + be converted to absolute paths by prepending the root_dir path parsed from + the command line, or its default value. + + Args: + files: a list of file path strings + + Returns: + A modified copy of the files list where any any path in a skipped + directory is removed, and all paths have been made absolute. + """ + newfiles = [] + for pathname in files: + if any(x in pathname for x in SKIPPED_DIRS): + continue + newfiles.append(pathname) + for idx, pathname in enumerate(newfiles): + if not os.path.isabs(pathname): + newfiles[idx] = os.path.join(ARGS.rootdir, pathname) + return newfiles + + +def get_files(extensions, ARGS): + """Generates a list of paths whose boilerplate should be verified. + + If a list of file names has been provided on the command line, it will be + treated as the initial set to search. Otherwise, all paths within rootdir + will be discovered and used as the initial set. + + Once the initial set of files is identified, it is normalized via + normalize_files() and further stripped of any file name whose extension is + not in extensions. + + Args: + extensions: a list of file extensions indicating which file types + should have their boilerplate verified + + Returns: + A list of absolute file paths + """ + files = [] + if ARGS.filenames: + files = ARGS.filenames + else: + for root, dirs, walkfiles in os.walk(ARGS.rootdir): + # don't visit certain dirs. This is just a performance improvement + # as we would prune these later in normalize_files(). But doing it + # cuts down the amount of filesystem walking we do and cuts down + # the size of the file list + for dpath in SKIPPED_DIRS: + if dpath in dirs: + dirs.remove(dpath) + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + files = normalize_files(files) + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = get_file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + + +def get_regexs(): + """Builds a map of regular expressions used in boilerplate validation. + + There are two scenarios where these regexes are used. The first is in + validating the date referenced is the boilerplate, by ensuring it is an + acceptable year. The second is in identifying non-boilerplate elements, + like shebangs and compiler hints that should be ignored when validating + headers. + + Returns: + A map of compiled regular expression objects, keyed by mnemonic. + """ + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the + # real thing + regexs["year"] = re.compile('YEAR') + # dates can be 2014, 2015, 2016 or 2017, company holder names can be + # anything + regexs["date"] = re.compile('(2014|2015|2016|2017|2018)') + # strip // +build \n\n build constraints + regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", + re.MULTILINE) + # strip #!.* from shell/python scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + return regexs + + +def main(args): + """Identifies and verifies files that should have the desired boilerplate. + + Retrieves the lists of files to be validated and tests each one in turn. + If all files contain correct boilerplate, this function terminates + normally. Otherwise it prints the name of each non-conforming file and + exists with a non-zero status code. + """ + regexs = get_regexs() + refs = get_refs(args) + filenames = get_files(refs.keys(), args) + nonconforming_files = [] + for filename in filenames: + if not has_valid_header(filename, refs, regexs): + nonconforming_files.append(filename) + if nonconforming_files: + print('%d files have incorrect boilerplate headers:' % len( + nonconforming_files)) + for filename in sorted(nonconforming_files): + print(os.path.relpath(filename, args.rootdir)) + sys.exit(1) + + +if __name__ == "__main__": + ARGS = get_args() + main(ARGS) diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..0f0aa9a --- /dev/null +++ b/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "credentials_file_path" { + description = "The path to the GCP credentials" +} + +variable "application_name" { + description = "The application to fetch secrets for" +} + +variable "env" { + description = "The environment to fetch secrets for" +} + +variable "secret" { + description = "The name of the secret to fetch" +} + +variable "shared" { + description = "Will we fetch the secret from the shared bucket instead of an application-specific bucket?" + type = "string" + default = "false" +}