diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index bd7b3c057b29..8068a9179a28 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -28,7 +28,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v6 + - uses: actions/stale@v7 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This pull request has been marked as stale due to 60 days of inactivity. It will be closed in 1 week if no further activity occurs. If you think that’s incorrect or this pull request requires a review, please simply write any comment. If closed, you can revive the PR at any time and @mention a reviewer or discuss it on the dev@beam.apache.org list. Thank you for your contributions.' diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 000000000000..6091765fd627 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +steps: + - name: 'ubuntu' + entrypoint: 'bash' + args: + - '-c' + - | + apt-get update + export DEBIAN_FRONTEND=noninteractive + apt-get install -y software-properties-common + add-apt-repository -y ppa:deadsnakes/ppa && apt update + apt install -y python3.8 python3-pip + cd playground/infrastructure + pip install -r requirements.txt + export \ + BEAM_ROOT_DIR="../../" \ + SDK_CONFIG="../../playground/sdks.yaml" \ + BEAM_EXAMPLE_CATEGORIES="../categories.yaml" \ + SERVER_ADDRESS="https://backend-${_SDK}-beta-dot-apache-beam-testing.appspot.com" \ + BEAM_USE_WEBGRPC=yes \ + BEAM_CONCURRENCY=4 + python3 ci_cd.py --step ${_STEP} --sdk SDK_${_SDK} --origin ${_ORIGIN} \ + --subdirs ${_SUBDIRS} + + env: + - 'ORIGIN=${_ORIGIN}' + - 'SDK=${_SDK}' + - 'STEP=${_STEP}' + - 'SUBDIRS=${_SUBDIRS}' + - 'GOOGLE_CLOUD_PROJECT=$PROJECT_ID' + +substitutions: + _STEP: CD + _SDK: GO + _ORIGIN: PG_EXAMPLES + _SUBDIRS: "../../learning/katas ../../examples ../../sdks" + +# This option enables writing logs to Cloud Logging +options: + logging: CLOUD_LOGGING_ONLY + +timeout: 3600s \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/README.md b/learning/tour-of-beam/terraform-v2/README.md new file mode 100644 index 000000000000..71215f4408de --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/README.md @@ -0,0 +1,79 @@ + + +# Overview + +This project deploys backend infrastructure for Tour of Beam as GCP Cloud Functions using Terraform. + +# Requirements + +## Development requirements + +- [Active GCP project with billing enabled](https://developers.google.com/workspace/guides/create-project) +- [Existing bucket to store Terraform state](https://cloud.google.com/storage/docs/creating-buckets) +- [Existing service account with the following roles](https://cloud.google.com/iam/docs/creating-managing-service-accounts): + - Cloud Functions Admin + - Project IAM Admin + - Service Account Admin + - Service Account User + - Storage Admin + - Storage Object Admin + +#### Configuring your environment + +Steps below will: +1. Enable required APIs for the project +2. Create service account and assign required IAM roles to it (service account to run the function with) +3. Create bucket to archive and store source code +4. Create cloud functions to each function defined in backend source code + + +```bash +# Create environment directory per your requirements/policy +mkdir environments/dev +cd ../environments/dev +# Create new configuration to auth to GCP Project +gcloud init +# Acquire new user credentials to use for Application Default Credentials +gcloud auth application-default login +# Initiliaze and run terraform +terraform init +terraform plan +terraform apply +``` + + +### Sample usage + +Entry point: list sdk names +``` +$ curl -X GET https://$REGION-$PROJECT_ID.cloudfunctions.net/getSdkList | json_pp +``` +[response](./samples/api/get_sdk_list.json) + +Get content tree by sdk name (SDK name == SDK id) +``` +$ curl -X GET 'https://$REGION-$PROJECT_ID.cloudfunctions.net/getContentTree?sdk=python' +``` +[response](./samples/api/get_content_tree.json) + + +Get unit content tree by sdk name and unitId +``` +$ curl -X GET 'https://$REGION-$PROJECT_ID.cloudfunctions.net/getContentTree?sdk=python&id=challenge1' +``` +[response](./samples/api/get_unit_content.json) \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/backend_state.tf b/learning/tour-of-beam/terraform-v2/backend_state.tf new file mode 100644 index 000000000000..41cb78351a45 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/backend_state.tf @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +terraform { + backend "gcs" { + prefix = "terraform-state" + } + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.40.0" + } + } +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/main.tf b/learning/tour-of-beam/terraform-v2/main.tf new file mode 100644 index 000000000000..ed3021552211 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/main.tf @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +provider "google" { + project = var.project_id +} + +module "iam" { + source = "./modules/iam" + project_id = var.project_id + service_account_id = var.service_account_id + depends_on = [module.api_enable] +} + +module "buckets" { + source = "./modules/buckets" + project_id = var.project_id + bucket_name = var.bucket_name + depends_on = [module.iam, module.api_enable] +} + +module "api_enable" { + source = "./modules/api_enable" + project_id = var.project_id +} + +module "cloud_functions" { + source = "./modules/cloud_functions" + region = var.region + project_id = var.project_id + bucket_name = var.bucket_name + service_account_id = module.iam.service-account-email + source_archive_bucket = module.buckets.functions-bucket-name + source_archive_object = module.buckets.function-bucket-object + depends_on = [module.buckets, module.iam, module.api_enable] +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf new file mode 100644 index 000000000000..d7ff91fffd9f --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +resource "google_project_service" "required_services" { + project = var.project_id + for_each = toset([ + "cloudbuild", + "cloudfunctions", + "cloudresourcemanager", + "iam", + ]) + service = "${each.key}.googleapis.com" + disable_on_destroy = false +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf new file mode 100644 index 000000000000..fef4fbca2c2b --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf new file mode 100644 index 000000000000..580abf0655d5 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +resource "google_storage_bucket" "cloud_functions_bucket" { + name = var.bucket_name + location = var.location + project = var.project_id + storage_class = "STANDARD" +} + +resource "google_storage_bucket_object" "zip" { + # Use an MD5 here. If there's no changes to the source code, this won't change either. + # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing + # a redeployment when it has! + name = "${data.archive_file.source_code.output_md5}.zip" + bucket = google_storage_bucket.cloud_functions_bucket.name + source = data.archive_file.source_code.output_path + content_type = "application/zip" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf new file mode 100644 index 000000000000..e2f16108f54a --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +output "functions-bucket-id" { + value = google_storage_bucket.cloud_functions_bucket.id +} + +output "functions-bucket-name" { + value = google_storage_bucket.cloud_functions_bucket.name +} + +output "function-bucket-object" { + value = google_storage_bucket_object.zip.name +} diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf new file mode 100644 index 000000000000..d60fee33091e --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#Generates archive of source code +variable "bucket_name" { + description = "The bucket name to store functions' source code" +} + +variable "location" { + description = "Cloud Functions bucket Region" + default = "us-central1" +} + +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +data "archive_file" "source_code" { + type = "zip" + source_dir = "../backend" + output_path = "/tmp/backend.zip" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf new file mode 100644 index 000000000000..b0da7a27065b --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +resource "google_cloudfunctions_function" "cloud_function" { + count = length(var.entry_point_names) + name = "${var.entry_point_names[count.index]}" + runtime = "go116" + available_memory_mb = 128 + project = var.project_id + service_account_email = var.service_account_id + source_archive_bucket = var.source_archive_bucket + source_archive_object = var.source_archive_object + region = var.region + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered + entry_point = var.entry_point_names[count.index] + + environment_variables = { + DATASTORE_PROJECT_ID="test-cloud-func-deploy" + TOB_MOCK=1 + } + +} + +# Create IAM entry so all users can invoke the function +resource "google_cloudfunctions_function_iam_member" "invoker" { + count = length(google_cloudfunctions_function.cloud_function) + project = google_cloudfunctions_function.cloud_function[count.index].project + region = google_cloudfunctions_function.cloud_function[count.index].region + cloud_function = google_cloudfunctions_function.cloud_function[count.index].name + + role = "roles/cloudfunctions.invoker" + member = "allUsers" + + depends_on = [google_cloudfunctions_function.cloud_function] +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf new file mode 100644 index 000000000000..b00710d1dda3 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +output "cloud-function-trigger-url" { + value = google_cloudfunctions_function.cloud_function.*.https_trigger_url +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf new file mode 100644 index 000000000000..abd7cdc955f2 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "region" {} +variable "project_id" {} +variable "service_account_id" { + description = "Name of SA to run Cloud Function" +} +variable "bucket_name" {} +variable "source_archive_bucket" {} +variable "source_archive_object" {} + +variable "entry_point_names" { + type = list + default = ["getSdkList", "getContentTree", "getUnitContent"] +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf new file mode 100644 index 000000000000..6341e5e774d2 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +resource "google_service_account" "cloud_function_sa" { + account_id = var.service_account_id + display_name = "Service Account to run Cloud Functions" +} + +resource "google_project_iam_member" "terraform_service_account_roles" { + for_each = toset([ + "roles/cloudfunctions.admin", "roles/storage.objectViewer", + "roles/iam.serviceAccountUser" + ]) + role = each.key + member = "serviceAccount:${google_service_account.cloud_function_sa.email}" + project = var.project_id +} + diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf new file mode 100644 index 000000000000..5971958aa03b --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +output "service-account-email" { + value = google_service_account.cloud_function_sa.email +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf new file mode 100644 index 000000000000..1dcceda523bb --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +variable "service_account_id" { + description = "The ID of the service account responsible for running Google Cloud functions" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/outputs.tf b/learning/tour-of-beam/terraform-v2/outputs.tf new file mode 100644 index 000000000000..a1e2ccb7583b --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/outputs.tf @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +output "service-account-email" { + value = module.iam.service-account-email +} + +output "cloud-function-trigger-url" { + value = module.cloud_functions.cloud-function-trigger-url +} + +output "functions-bucket-name" { + value = module.buckets.functions-bucket-name +} + +output "function-bucket-object" { + value = module.buckets.function-bucket-object +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/variables.tf b/learning/tour-of-beam/terraform-v2/variables.tf new file mode 100644 index 000000000000..8d52d1a7223b --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/variables.tf @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "bucket_name" { + description = "The bucket name that will store functions' source code" +} + +variable "project_id" { + type = string + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +variable "service_account_id" { + type = string + description = "The ID of the service account responsible for running Google Cloud functions" +} + +variable "region" { + default = "us-central1" +} \ No newline at end of file diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml new file mode 100644 index 000000000000..e2577b66a00b --- /dev/null +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +steps: +# This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment + - name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' + entrypoint: "/bin/bash" + args: + - "-c" + - | + chmod +x playground/infrastructure/cloudbuild/env_init.sh \ + && ./playground/infrastructure/cloudbuild/env_init.sh + mkdir playground/terraform/environment/$_ENVIRONMENT_NAME + printf '%s\n' \ + 'project_id = "$PROJECT_ID"' \ + 'network_name = "$_NETWORK_NAME"' \ + 'gke_name = "$_GKE_NAME"' \ + 'region = "$_PLAYGROUND_REGION"' \ + 'location = "$_PLAYGROUND_LOCATION"' \ + 'state_bucket = "$_STATE_BUCKET"' \ + > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars + printf \ + 'bucket = "$_STATE_BUCKET"'\ + > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend + ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" + +# This option enables writing logs to Cloud Logging +options: + logging: CLOUD_LOGGING_ONLY + +timeout: 3600s \ No newline at end of file diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml new file mode 100644 index 000000000000..42965dffcab4 --- /dev/null +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +steps: + # This stage uses pre-built Docker container to run gradle task for Playground pre-config and deployment to GKE + - name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' + entrypoint: "/bin/bash" + args: + - "-c" + - | + chmod +x playground/infrastructure/cloudbuild/env_init.sh \ + && ./playground/infrastructure/cloudbuild/env_init.sh + mkdir playground/terraform/environment/$_ENVIRONMENT_NAME + printf '%s\n' \ + 'project_id = "$PROJECT_ID"' \ + 'network_name = "$_NETWORK_NAME"' \ + 'gke_name = "$_GKE_NAME"' \ + 'region = "$_PLAYGROUND_REGION"' \ + 'location = "$_PLAYGROUND_LOCATION"' \ + 'state_bucket = "$_STATE_BUCKET"' \ + > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars + printf \ + 'bucket = "$_STATE_BUCKET"'\ + > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend + gcloud container clusters get-credentials --region '$_PLAYGROUND_LOCATION' '$_GKE_NAME' --project '$PROJECT_ID' + ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" + ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ + -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" + +# This option enables writing logs to Cloud Logging +options: + logging: CLOUD_LOGGING_ONLY + +timeout: 7600s \ No newline at end of file diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh new file mode 100644 index 000000000000..c612472433bd --- /dev/null +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apt-get update > /dev/null + +# Install dependencies +apt-get install -y build-essential unzip apt-transport-https ca-certificates curl software-properties-common gnupg2 wget > /dev/null + +# Install Docker +curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" > /dev/null +apt update > /dev/null && apt install -y docker-ce > /dev/null + +#Install Helm +curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 > /dev/null +chmod +x get_helm.sh && ./get_helm.sh > /dev/null + +# Install Terraform +wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ +| tee /etc/apt/sources.list.d/hashicorp.list +apt update -y > /dev/null && apt install -y terraform > /dev/null + +# Install kubectl +curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ +&& chmod +x ./kubectl \ +&& mv ./kubectl /usr/local/bin/kubectl + +# Install jdk +apt-get install openjdk-11-jdk -y > /dev/null \ No newline at end of file diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index dca9a8dc5024..b73752ba0f17 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -464,6 +464,7 @@ dns_name: ${dns_name} """) } } + helm { val playground by charts.creating { chartName.set("playground") @@ -475,26 +476,24 @@ helm { } } } + task ("gkebackend") { group = "deploy" val init = tasks.getByName("terraformInit") - val apply = tasks.getByName("terraformApplyInf") - val indexcreate = tasks.getByName("indexcreate") val takeConfig = tasks.getByName("takeConfig") + val back = tasks.getByName("pushBack") val front = tasks.getByName("pushFront") - val push = tasks.getByName("pushBack") + val indexcreate = tasks.getByName("indexcreate") val helm = tasks.getByName("helmInstallPlayground") dependsOn(init) - dependsOn(apply) dependsOn(takeConfig) - dependsOn(push) + dependsOn(back) dependsOn(front) dependsOn(indexcreate) dependsOn(helm) - apply.mustRunAfter(init) - takeConfig.mustRunAfter(apply) - push.mustRunAfter(takeConfig) - front.mustRunAfter(push) + takeConfig.mustRunAfter(init) + back.mustRunAfter(takeConfig) + front.mustRunAfter(back) indexcreate.mustRunAfter(front) helm.mustRunAfter(indexcreate) } diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf new file mode 100644 index 000000000000..3117134ffcc9 --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +resource "google_service_account" "cloudbuild_service_account_id" { + account_id = var.cloudbuild_service_account_id + display_name = var.cloudbuild_service_account_id + description = "The service account cloud build will use to deploy Playground" +} + +// Provision IAM roles to the IaC service account required to build and provision resources +resource "google_project_iam_member" "cloud_build_roles" { + for_each = toset([ + "roles/appengine.appAdmin", + "roles/appengine.appCreator", + "roles/artifactregistry.admin", + "roles/redis.admin", + "roles/compute.admin", + "roles/iam.serviceAccountCreator", + "roles/container.admin", + "roles/servicemanagement.quotaAdmin", + "roles/iam.securityAdmin", + "roles/iam.serviceAccountUser", + "roles/datastore.indexAdmin", + "roles/storage.admin", + "roles/logging.logWriter" + ]) + role = each.key + member = "serviceAccount:${google_service_account.cloudbuild_service_account_id.email}" + project = var.project_id +} \ No newline at end of file diff --git a/playground/terraform/environment/beta/state.tfbackend b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf similarity index 94% rename from playground/terraform/environment/beta/state.tfbackend rename to playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf index b2c574de8e4b..c23ddd6f9bf6 100644 --- a/playground/terraform/environment/beta/state.tfbackend +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -bucket = "pg-second" +provider "google" { + project = var.project_id +} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf new file mode 100644 index 000000000000..d44822e37ba5 --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +// Provision the required Google Cloud services +resource "google_project_service" "required_services" { + project = var.project_id + for_each = toset([ + "cloudresourcemanager", + "cloudbuild", + "appengine", + "artifactregistry", + "compute", + "redis", + "iam", + ]) + + service = "${each.key}.googleapis.com" + disable_on_destroy = false +} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/terraform.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/terraform.tf new file mode 100644 index 000000000000..e7baa96ccd74 --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/terraform.tf @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +terraform { + backend "gcs" { + prefix = "01.setup" + } + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0.0" + } + } +} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf new file mode 100644 index 000000000000..3ab4de68af34 --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "project_id" { + type = string + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +variable "cloudbuild_service_account_id" { + type = string + description = "The ID of the cloud build service account responsible for provisioning Google Cloud resources" + default = "playground-cloudbuild-sa" +} diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf new file mode 100644 index 000000000000..c23ddd6f9bf6 --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +provider "google" { + project = var.project_id +} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf new file mode 100644 index 000000000000..e0da73116ef8 --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +terraform { + backend "gcs" { + prefix = "02.builders" + } + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.40.0" + } + } +} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf new file mode 100644 index 000000000000..335cfe12edec --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +data "google_service_account" "cloudbuild_sa" { + account_id = var.cloudbuild_service_account_id +} + +resource "google_cloudbuild_trigger" "playground_infrastructure" { + name = var.infra_trigger_name + location = var.region + project = var.project_id + + description = "Builds the base image and then runs cloud build config file to deploy Playground infrastructure" + + github { + owner = var.github_repository_owner + name = var.github_repository_name + push { + branch = var.github_repository_branch + } + } + + substitutions = { + _PLAYGROUND_REGION : var.playground_region + _PLAYGROUND_LOCATION : var.playground_zone + _ENVIRONMENT_NAME : var.playground_environment_name + _DNS_NAME : var.playground_dns_name + _NETWORK_NAME : var.playground_network_name + _GKE_NAME : var.playground_gke_name + _STATE_BUCKET : var.state_bucket + } + + filename = "playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml" + + service_account = data.google_service_account.cloudbuild_sa.id +} + +resource "google_cloudbuild_trigger" "playground_to_gke" { + name = var.gke_trigger_name + location = var.region + project = var.project_id + + description = "Builds the base image and then runs cloud build config file to deploy Playground to GKE" + + github { + owner = var.github_repository_owner + name = var.github_repository_name + push { + branch = var.github_repository_branch + } + } + + substitutions = { + _PLAYGROUND_REGION : var.playground_region + _PLAYGROUND_LOCATION : var.playground_zone + _ENVIRONMENT_NAME : var.playground_environment_name + _DNS_NAME : var.playground_dns_name + _NETWORK_NAME : var.playground_network_name + _GKE_NAME : var.playground_gke_name + _STATE_BUCKET : var.state_bucket + _TAG : var.image_tag + _DOCKER_REPOSITORY_ROOT : var.docker_repository_root + _SDK_TAG : var.sdk_tag + } + + filename = "playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml" + + service_account = data.google_service_account.cloudbuild_sa.id +} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf new file mode 100644 index 000000000000..b8462217422b --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "project_id" { + type = string + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +variable "region" { + type = string + description = "The Google Cloud Platform (GCP) region in which to provision resources" +} + +variable "infra_trigger_name" { + type = string + description = "The name of the trigger that will deploy Playground infrastructure" + default = "playground-infrastructure-trigger" +} + +variable "gke_trigger_name" { + type = string + description = "The name of the trigger that will deploy Playground to GKE" + default = "playground-to-gke-trigger" +} + +variable "cloudbuild_service_account_id" { + type = string + description = "The ID of the cloud build service account responsible for provisioning Google Cloud resources" + default = "playground-cloudbuild-sa" +} + +variable "github_repository_name" { + type = string + description = "The name of the GitHub repository. For example the repository name for https://github.com/example/foo is 'foo'." +} + +variable "github_repository_owner" { + type = string + description = "Owner of the GitHub repository. For example the owner for https://github.com/example/foo is 'example'." +} + +variable "github_repository_branch" { + type = string + description = "The GitHub repository branch regex to match cloud build trigger" +} + +variable "playground_environment_name" { + description = "Environment where to deploy Playground. Located in playground/terraform/environment/{environment_name}. E.g. test, dev, prod" +} + +variable "playground_dns_name" { + description = "DNS record name for Playground website. More details: https://github.com/apache/beam/blob/master/playground/terraform/README.md#deploy-playground-infrastructure" +} + +variable "playground_network_name" { + description = "The Google Cloud Platform (GCP) VPC Network Name for Playground deployment" +} + +variable "playground_gke_name" { + description = "Playground GKE Cluster name in Google Cloud Platform (GCP)" +} + +variable "state_bucket" { + description = "The Google Cloud Platform (GCP) GCS bucket name for Beam Playground temp files and Terraform state" +} + +variable "image_tag" { + description = "The tag name for docker images of Playground containers" +} + +variable "docker_repository_root" { + description = "The name of Google Cloud Platform (GCP) Artifact Registry Repository where Playground images will be saved to" +} + +variable "playground_region" { + description = "The Google Cloud Platform (GCP) region (For example: us-central1) where playground infrastructure will be deployed to" +} + +variable "playground_zone" { + description = "The Google Cloud Platform (GCP) zone (For example: us-central1-b) where playground infrastructure will be deployed to" +} + +variable "sdk_tag" { + description = "Apache Beam Golang and Python images SDK tag. See more: https://hub.docker.com/r/apache/beam_python3.7_sdk/tags and https://hub.docker.com/r/apache/beam_go_sdk" +} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md new file mode 100644 index 000000000000..fdb2704460e2 --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -0,0 +1,119 @@ + + +# Beam Playground Cloud Build Setup + +This directory organizes Infrastructure-as-Code to provision dependent resources and set up Cloud Build for Beam Playground. + +Playground deployment main data and scripts can be found by next locations: +- [Playground deployment main directory](https://github.com/apache/beam/blob/master/playground/) +- [Playground deployment main README file](https://github.com/apache/beam/blob/master/playground/terraform/README.md) + +## Requirements: + +- [GCP project](https://cloud.google.com/resource-manager/docs/creating-managing-projects) +- [GCP User account](https://cloud.google.com/appengine/docs/standard/access-control?tab=python) _(Note: You will find the instruction "How to create User account" for your new project)_
+ Ensure that the account has at least the following [IAM roles](https://cloud.google.com/iam/docs/understanding-roles): + - Service Account Admin + - Storage Admin + - Service Usage Admin + - Cloud Build Editor + - Security Admin + - Service Account User +- [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) +- An existing GCP Bucket to save Terraform state - `state-bucket` +- DNS name for your Playground deployment instance +- [Terraform](https://www.terraform.io/) +- [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally + +## 1. Set up the Google Cloud Build for your GCP project + +The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provisions dependencies required to set up Cloud Build for Playground: +- Required API services +- Cloud Build service account - `playground-cloudbuild-sa` +- IAM permissions for Cloud Build service account - `playground-cloudbuild-sa` + +#### To execute the module: + +**Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. + +1. Run commands: + + +```console +# Set environment variable for state bucket +export STATE_BUCKET="state-bucket" + +# Create a new authentication configuration for GCP Project with the created user account +gcloud init + +# Command imports new user account credentials into Application Default Credentials +gcloud auth application-default login + +# Navigate to 01.setup directory +cd playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/ + +# Run terraform commands +terraform init -backend-config="bucket=$STATE_BUCKET" +terraform apply -var="project_id=$(gcloud config get-value project)" +``` + +## 2. Connect Apache Beam GitHub repository and GCP Cloud Build + +**Note:** Ensure correct `region` is set in [Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page before proceeding further. + +Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github) to connect Apache Beam GitHub repository and GCP Cloud Build. + +## 3. Set up the Google Cloud Build triggers + +The `playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders` provisions: +- Cloud Build triggers to build and deploy Beam Playground + +#### To execute the module + +``` +# Navigate to playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders directory +cd ../02.builders + +# Run terraform commands +terraform init -backend-config="bucket=$STATE_BUCKET" +terraform apply -var="project_id=$(gcloud config get-value project)" +``` + +## 4. Run Cloud Build `Playground-infrastructure-trigger` to deploy Playground infrastructure + +1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (In our example: us-central1). +2. Open Trigger: `Playground-infrastructure-trigger`. +3. Scroll down to `Source` - `Repository` to ensure that Apache Beam GitHub repository is connected. + - Click on drop-down menu and press `CONNECT NEW REPOSITORY` in case it was not automatically connected. +4. Click `Save` and Run the trigger `Playground-infrastructure-trigger`. + +5. Once Playground infrastructure has been deployed, please navigate to + [Playground deployment README](https://github.com/apache/beam/tree/master/playground/terraform#deploy-playground-infrastructure) and execute step #2: + `Add following DNS A records for the discovered static IP address` expanding use of variable `DNS_NAME`. + +## 5. Run Cloud Build `Playground-to-gke-trigger` to deploy Playground to GKE + +1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (In our example: us-central1). +2. Open Trigger: `Playground-to-gke-trigger`. +3. Scroll down to `Source` - `Repository` to ensure that Apache Beam GitHub repository is connected. + - Click on drop-down menu and press `CONNECT NEW REPOSITORY` in case it was not automatically connected. +4. Click `Save` and Run the trigger `Playground-to-gke-trigger`. + +## 6. Validate Playground deployment + +Once Playground has been deployed to GKE, please navigate to [Validation](https://github.com/apache/beam/tree/master/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file