diff --git a/cloudbuild_playground_backend.yml b/cloudbuild_playground_backend.yml new file mode 100644 index 000000000000..03f637b8aa89 --- /dev/null +++ b/cloudbuild_playground_backend.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + + - name: 'openjdk:11' + args: ['-c', './gradlew playground:backend:precommit'] + # This step builds the base image for Beam Playground backend builds +# - name: 'gcr.io/cloud-builders/docker' +# args: +# - 'build' +# - '--tag=us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# - '-f' +# - 'gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend' +# - '.' +# +# # This step builds Beam Playground backend container for golang +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:go:docker' +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:java:docker' +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:python:docker' +# +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:router:docker' +# +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:scio:docker' +# +#substitutions: +# _DOCKERTAG: ${GITHUB_SHA} +# _ARTIFACT_REGISTRY_REPOSITORY_ID: playground-repository +# +#images: +# - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# +#timeout: 3600s \ No newline at end of file diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml new file mode 100644 index 000000000000..a57df39c33b2 --- /dev/null +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + + - name: 'golang' + args: ['./gradlew playground:backend:precommit'] + # This step builds the base image for Beam Playground backend builds +# - name: 'gcr.io/cloud-builders/docker' +# args: +# - 'build' +# - '--tag=us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# - '-f' +# - 'gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend' +# - '.' +# +# # This step builds Beam Playground backend container for golang +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:go:docker' +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:java:docker' +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:python:docker' +# +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:router:docker' +# +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:scio:docker' +# +#substitutions: +# _DOCKERTAG: ${GITHUB_SHA} +# _ARTIFACT_REGISTRY_REPOSITORY_ID: playground-repository +# +#images: +# - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# +#timeout: 3600s \ No newline at end of file diff --git a/gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend b/gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend new file mode 100644 index 000000000000..a0f935287a64 --- /dev/null +++ b/gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend @@ -0,0 +1,41 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +FROM gcr.io/cloud-builders/gcloud + +ARG GO_VERSION=1.19.2 +ENV PATH="${PATH}:/usr/local/go/bin" + +RUN apt-get update >/dev/null + +# Install Docker +# See: https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script +RUN curl -fsSL https://get.docker.com -o get-docker.sh +RUN sh ./get-docker.sh >/dev/null + +# Install Go +# See: https://go.dev/doc/install +RUN curl -OL https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz +RUN sha256sum go$GO_VERSION.linux-amd64.tar.gz +RUN tar -C /usr/local -xvf go$GO_VERSION.linux-amd64.tar.gz +RUN rm go$GO_VERSION.linux-amd64.tar.gz +RUN go version + +# Install Java +RUN apt-get install openjdk-11-jdk -y >/dev/null +RUN java -version \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/README.md b/learning/tour-of-beam/terraform-v2/README.md new file mode 100644 index 000000000000..17ac6b6eb7f8 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/README.md @@ -0,0 +1,60 @@ +# [Tour of Beam] Deploying backend infrastructure as GCP Cloud Function with GCP, Terraform and Cloud Build +This tutorial explains how to create Tour of Beam backend infrastructure as a GCP Cloud Functions using Infrastructure as a Code (IaC) and GitOps methodoliges. + + +### Deployment +Prerequisites: +- Active GCP project with billing enabled +- Existing bucket to store Terraform state (name to be declared in backend.tf) +- Existing service account with the following roles: + - Cloud Functions Admin + - Project IAM Admin + - Service Account Admin + - Service Account User + - Storage Admin + - Storage Object Admin +- Exported JSON Key of service account above + +#### Configuring your environment + +Steps below will: +1. Enable required APIs for the project +2. Create service account and assign required IAM roles to it (service account to run the function with) +3. Create bucket to archive and store source code +4. Create cloud functions to each function defined in backend source code + + +```bash +# Create environment directory per your requirements/policy +mkdir environments/dev +cd ../environments/dev +# Import exported JSON Key using +export GOOGLE_APPLICATION_CREDENTIALS = '..key path...' +# Initiliaze and run terraform +terraform init +terraform plan +terraform apply +terraform destroy +``` + + +### Sample usage + +Entry point: list sdk names +``` +$ curl -X GET https://$REGION-$PROJECT_ID.cloudfunctions.net/getSdkList | json_pp +``` +[response](./samples/api/get_sdk_list.json) + +Get content tree by sdk name (SDK name == SDK id) +``` +$ curl -X GET 'https://$REGION-$PROJECT_ID.cloudfunctions.net/getContentTree?sdk=python' +``` +[response](./samples/api/get_content_tree.json) + + +Get unit content tree by sdk name and unitId +``` +$ curl -X GET 'https://$REGION-$PROJECT_ID.cloudfunctions.net/getContentTree?sdk=python&id=challenge1' +``` +[response](./samples/api/get_unit_content.json) \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/backend_state.tf b/learning/tour-of-beam/terraform-v2/backend_state.tf new file mode 100644 index 000000000000..85723beb13a8 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/backend_state.tf @@ -0,0 +1,12 @@ +terraform { + backend "gcs" { + bucket = "tour-of-beam-backend-tfstate-bucket" + prefix = "terraform-state" + } + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.40.0" + } + } +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/main.tf b/learning/tour-of-beam/terraform-v2/main.tf new file mode 100644 index 000000000000..da89eb863e1d --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/main.tf @@ -0,0 +1,33 @@ +provider "google" { + project = var.project_id +} + +module "iam" { + source = "./modules/iam" + project_id = var.project_id + service_account_id = var.service_account_id + depends_on = [module.api_enable] +} + +module "buckets" { + source = "./modules/buckets" + project_id = var.project_id + bucket_name = var.bucket_name + depends_on = [module.iam, module.api_enable] +} + +module "api_enable" { + source = "./modules/api_enable" + project_id = var.project_id +} + +module "cloud_functions" { + source = "./modules/cloud_functions" + region = var.region + project_id = var.project_id + bucket_name = var.bucket_name + service_account_id = module.iam.service-account-email + source_archive_bucket = module.buckets.functions-bucket-name + source_archive_object = module.buckets.function-bucket-object + depends_on = [module.buckets, module.iam, module.api_enable] +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf new file mode 100644 index 000000000000..a0fb23a12a95 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf @@ -0,0 +1,27 @@ +# Enable API for Cloud Build +resource "google_project_service" "cloud_build" { + project = var.project_id + service = "cloudbuild.googleapis.com" + disable_on_destroy = false +} + +# Enable API for Cloud Function +resource "google_project_service" "cloud_function" { + project = var.project_id + service = "cloudfunctions.googleapis.com" + disable_on_destroy = false +} + +# Enable API for Resource Manager +resource "google_project_service" "resource_manager" { + project = var.project_id + service = "cloudresourcemanager.googleapis.com" + disable_on_destroy = false +} + +# Enable API for IAM +resource "google_project_service" "iam" { + project = var.project_id + service = "iam.googleapis.com" + disable_on_destroy = false +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf new file mode 100644 index 000000000000..7488e265382a --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf @@ -0,0 +1,3 @@ +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf new file mode 100644 index 000000000000..85f100e4672d --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -0,0 +1,16 @@ +resource "google_storage_bucket" "cloud_functions_bucket" { + name = var.bucket_name + location = var.location + project = var.project_id + storage_class = "STANDARD" +} + +resource "google_storage_bucket_object" "zip" { + # Use an MD5 here. If there's no changes to the source code, this won't change either. + # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing + # a redeployment when it has! + name = "${data.archive_file.source_code.output_md5}.zip" + bucket = google_storage_bucket.cloud_functions_bucket.name + source = data.archive_file.source_code.output_path + content_type = "application/zip" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf new file mode 100644 index 000000000000..4841dc829edb --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf @@ -0,0 +1,11 @@ +output "functions-bucket-id" { + value = google_storage_bucket.cloud_functions_bucket.id +} + +output "functions-bucket-name" { + value = google_storage_bucket.cloud_functions_bucket.name +} + +output "function-bucket-object" { + value = google_storage_bucket_object.zip.name +} diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf new file mode 100644 index 000000000000..e3522f2c0c39 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -0,0 +1,19 @@ +#Generates archive of source code +variable "bucket_name" { + description = "The bucket name to store functions' source code" +} + +variable "location" { + description = "Cloud Functions bucket Region" + default = "us-central1" +} + +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +data "archive_file" "source_code" { + type = "zip" + source_dir = "../backend" + output_path = "/tmp/backend.zip" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf new file mode 100644 index 000000000000..45ceb8c5f3b3 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -0,0 +1,35 @@ +resource "google_cloudfunctions_function" "cloud_function" { + count = length(var.entry_point_names) + name = "${var.entry_point_names[count.index]}" + runtime = "go116" + available_memory_mb = 128 + project = var.project_id + service_account_email = var.service_account_id + source_archive_bucket = var.source_archive_bucket + source_archive_object = var.source_archive_object + region = var.region + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered + entry_point = var.entry_point_names[count.index] + + environment_variables = { + DATASTORE_PROJECT_ID="test-cloud-func-deploy" + TOB_MOCK=1 + } + +} + +# Create IAM entry so all users can invoke the function +resource "google_cloudfunctions_function_iam_member" "invoker" { + count = length(google_cloudfunctions_function.cloud_function) + project = google_cloudfunctions_function.cloud_function[count.index].project + region = google_cloudfunctions_function.cloud_function[count.index].region + cloud_function = google_cloudfunctions_function.cloud_function[count.index].name + + role = "roles/cloudfunctions.invoker" + member = "allUsers" + + depends_on = [google_cloudfunctions_function.cloud_function] +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf new file mode 100644 index 000000000000..72b116a897e8 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf @@ -0,0 +1,3 @@ +output "cloud-function-trigger-url" { + value = google_cloudfunctions_function.cloud_function.*.https_trigger_url +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf new file mode 100644 index 000000000000..189b192e08ef --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -0,0 +1,13 @@ +variable "region" {} +variable "project_id" {} +variable "service_account_id" { + description = "Name of SA to run Cloud Function" +} +variable "bucket_name" {} +variable "source_archive_bucket" {} +variable "source_archive_object" {} + +variable "entry_point_names" { + type = list + default = ["getSdkList", "getContentTree", "getUnitContent"] +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf new file mode 100644 index 000000000000..148a78c4398e --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -0,0 +1,15 @@ +resource "google_service_account" "cloud_function_sa" { + account_id = var.service_account_id + display_name = "Service Account to run Cloud Functions" +} + +resource "google_project_iam_member" "terraform_service_account_roles" { + for_each = toset([ + "roles/cloudfunctions.admin", "roles/storage.objectViewer", + "roles/storage.objectCreator", "roles/iam.serviceAccountUser" + ]) + role = each.key + member = "serviceAccount:${google_service_account.cloud_function_sa.email}" + project = var.project_id +} + diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf new file mode 100644 index 000000000000..57285172fce6 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf @@ -0,0 +1,3 @@ +output "service-account-email" { + value = google_service_account.cloud_function_sa.email +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf new file mode 100644 index 000000000000..ac40f8352b60 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -0,0 +1,8 @@ +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +variable "service_account_id" { + description = "The ID of the service account responsible for running Google Cloud functions" +} + diff --git a/learning/tour-of-beam/terraform-v2/outputs.tf b/learning/tour-of-beam/terraform-v2/outputs.tf new file mode 100644 index 000000000000..ca0b2b553155 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/outputs.tf @@ -0,0 +1,15 @@ +output "service-account-email" { + value = module.iam.service-account-email +} + +output "cloud-function-trigger-url" { + value = module.cloud_functions.cloud-function-trigger-url +} + +output "functions-bucket-name" { + value = module.buckets.functions-bucket-name +} + +output "function-bucket-object" { + value = module.buckets.function-bucket-object +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/variables.tf b/learning/tour-of-beam/terraform-v2/variables.tf new file mode 100644 index 000000000000..55e10baf164e --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/variables.tf @@ -0,0 +1,23 @@ +variable "bucket_name" { + description = "The bucket name that will store functions' source code" +} + +variable "project_id" { + type = string + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +variable "iac_service_account_id" { + type = string + description = "The ID of the service account responsible for provisioning Google Cloud resources using Infrastructure-as-Code" + default = "terraform" +} + +variable "service_account_id" { + type = string + description = "The ID of the service account responsible for running Google Cloud functions" +} + +variable "region" { + default = "us-central1" +} \ No newline at end of file diff --git a/playground/terraform/environment/beta/state.tfbackend b/playground/terraform/environment/beta/state.tfbackend deleted file mode 100644 index 991c76536a5e..000000000000 --- a/playground/terraform/environment/beta/state.tfbackend +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -bucket = "beam_playground_terraform" diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 27da7a0598c6..403455457b7b 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -328,7 +328,7 @@ def get_portability_package_data(): 'needle>=0.5.0,<1', 'chromedriver-binary>=100,<101', # use a fixed major version of PIL for different python versions - 'pillow>=7.1.1,<8', + 'pillow>=7.1.1,<10', ], 'aws': ['boto3 >=1.9'], 'azure': [