From 8a3fa8549ac724c34a07619960bde73de8716634 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 14 Nov 2022 00:48:56 +0500 Subject: [PATCH 001/269] Commit related to apache #24007, #24006 --- .../cloudbuild/Dockerfile-base-image-worker | 56 +++++++++++++++++++ .../cloudbuild/cloudbuild_pg_infra.yaml | 53 ++++++++++++++++++ .../cloudbuild/cloudbuild_pg_to_gke.yaml | 56 +++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 playground/infrastructure/cloudbuild/Dockerfile-base-image-worker create mode 100644 playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml create mode 100644 playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml diff --git a/playground/infrastructure/cloudbuild/Dockerfile-base-image-worker b/playground/infrastructure/cloudbuild/Dockerfile-base-image-worker new file mode 100644 index 000000000000..419f40442fac --- /dev/null +++ b/playground/infrastructure/cloudbuild/Dockerfile-base-image-worker @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM gcr.io/google.com/cloudsdktool/google-cloud-cli + +ARG GO_VERSION=1.18 +ENV PATH="${PATH}:/usr/local/go/bin" + +RUN apt-get update >/dev/null +RUN apt-get -y install build-essential unzip + +# Install Docker +# See: https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script +RUN curl -fsSL https://get.docker.com -o get-docker.sh +RUN sh ./get-docker.sh >/dev/null + +# Install Go +# See: https://go.dev/doc/install +RUN curl -OL https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz +RUN sha256sum go$GO_VERSION.linux-amd64.tar.gz +RUN tar -C /usr/local -xvf go$GO_VERSION.linux-amd64.tar.gz +RUN rm go$GO_VERSION.linux-amd64.tar.gz +RUN go version + +#Install kubectl +RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ + && chmod +x ./kubectl \ + && mv ./kubectl /usr/local/bin/kubectl \ + && curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 \ + && chmod +x get_helm.sh && ./get_helm.sh > /dev/null + +#Install Terraform +RUN curl -OL https://releases.hashicorp.com/terraform/1.3.3/terraform_1.3.3_linux_amd64.zip +RUN unzip terraform_1.3.3_linux_amd64.zip +RUN mv terraform /usr/local/bin + +#Install GoLint +RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.50.1 + +# Install Java +RUN apt-get install openjdk-11-jdk -y >/dev/null +RUN java -version \ No newline at end of file diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml new file mode 100644 index 000000000000..13290a9a0322 --- /dev/null +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +steps: + # This step builds the image from Dockerfile that will be used as Docker machine for Beam Playground deployment + - name: 'gcr.io/cloud-builders/docker' + args: + - 'build' + - '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/builder:worker' + - '-f' + - 'playground/infrastructure/cloudbuild/Dockerfile-base-image-worker' + - '.' + + # This step uses Docker container from image in previous step to run gradle task for Playground Infrastructure deployment + - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/builder:worker' + entrypoint: "/bin/bash" + # Please set variables $_ENVIRONMENT_NAME and $_DNS_NAME in "substitutions" stage below + args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] + +substitutions: + # The name of the Google Cloud Artifact Registry + _ARTIFACT_REGISTRY_REPOSITORY_ID: "playground-repository" + # The name of your environment in beam/playground/terraform/environment/ + _ENVIRONMENT_NAME: "test" + # The name of your DNS for Playground + _DNS_NAME: "dev-playground.online" + +# Please change to your logs bucket +logsBucket: 'gs://pg-cloudbuild-logging-bucket' + +# Pushing built image to Artifact repository +images: + - '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/builder:worker' + +# This option is for writing logs to GCS bucket +options: + logging: GCS_ONLY + +timeout: 1800s \ No newline at end of file diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml new file mode 100644 index 000000000000..e4b2e0471341 --- /dev/null +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +steps: + # This step uses Docker container from our image to authenticate us in GKE + # Please set variable $_GKE_NAME in "substitutions" stage below + - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/builder:worker' + entrypoint: "gcloud" + args: + [ + 'container', + 'clusters', + 'get-credentials', + '$_GKE_NAME', + '--region=$LOCATION-a' + ] + + # This step uses Docker container from image in previous step to run gradle task for Playground to GKE deployment + - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/builder:worker' + entrypoint: "/bin/bash" + args: [ '-c', './gradlew playground:terraform:gkebackend -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Pdns-name="$_DNS_NAME"' ] + +substitutions: + # The name of the Google Cloud Artifact Registry + _ARTIFACT_REGISTRY_REPOSITORY_ID: "playground-repository" + # The name of your environment in beam/playground/terraform/environment/ + _ENVIRONMENT_NAME: "test" + # The name of your DNS for Playground + _DNS_NAME: "dev-playground.online" + # Docker image tag name + _TAG: "ruslan" + # The name of your GKE cluster (from terraform.tfvars) + _GKE_NAME: "playground-backend" + +# Please change to your logs bucket +logsBucket: 'gs://pg-cloudbuild-logging-bucket' + +# This option is for writing logs to GCS bucket +options: + logging: GCS_ONLY + +timeout: 3600s \ No newline at end of file From 6d130a49e27f9a012ab41e7e3be187bd17574177 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 14 Nov 2022 01:01:11 +0500 Subject: [PATCH 002/269] Commit related to apache #24007, #24006 --- .../cloudbuild-manual-setup/01.setup/iam.tf | 44 +++++++ .../01.setup/output.tf | 31 +++++ .../01.setup/provider.tf | 21 ++++ .../01.setup/services.tf | 32 +++++ .../01.setup/terraform.tf | 28 +++++ .../01.setup/variables.tf | 33 +++++ .../02.builders/output.tf | 25 ++++ .../02.builders/provider.tf | 21 ++++ .../02.builders/terraform.tf | 28 +++++ .../02.builders/triggers.tf | 66 ++++++++++ .../02.builders/variables.tf | 60 +++++++++ .../cloudbuild-manual-setup/README.md | 117 ++++++++++++++++++ 12 files changed, 506 insertions(+) create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/output.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/terraform.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/output.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/README.md diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf new file mode 100644 index 000000000000..74b32013bdc8 --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +resource "google_service_account" "cloudbuild_service_account_id" { + account_id = var.cloudbuild_service_account_id + display_name = var.cloudbuild_service_account_id + description = "The service account cloud build will use to deploy Playground" +} + +// Provision IAM roles to the IaC service account required to build and provision resources +resource "google_project_iam_member" "cloud_build_roles" { + for_each = toset([ + "roles/appengine.appAdmin", + "roles/appengine.appCreator", + "roles/artifactregistry.admin", + "roles/redis.admin", + "roles/compute.admin", + "roles/iam.serviceAccountCreator", + "roles/container.admin", + "roles/servicemanagement.quotaAdmin", + "roles/iam.securityAdmin", + "roles/iam.serviceAccountUser", + "roles/datastore.indexAdmin", + "roles/storage.admin", + "roles/logging.logWriter" + ]) + role = each.key + member = "serviceAccount:${google_service_account.cloudbuild_service_account_id.email}" + project = var.project +} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/output.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/output.tf new file mode 100644 index 000000000000..e7405c14ea29 --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/output.tf @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +output "cloudbuild_service_account_id" { + value = google_service_account.cloudbuild_service_account_id +} + +output "How_to_connect_github_repo_to_cloudbuild" { + value = < + +# Beam Playground Cloud Build Setup + +This directory organizes Infrastructure-as-Code to provision dependent resources and setup Cloud Build for Beam Playground. + +## Requirements: + +- [GCP project](https://cloud.google.com/) +- Enabled [Cloud Build API](https://cloud.google.com/apis/docs/getting-started#enabling_apis) +- [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) +- An existing Google Cloud Storage Bucket to save Terraform state - `state_bucket` +- And 2nd existing GCP Bucket to save Cloud build logs +- DNS name for your Playground deployment instance +- [Terraform](https://www.terraform.io/) +- [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally + +## Usage + +The folders are named according to the required order of execution. + +### 1. Setup the Google Cloud Build for your GCP project + +The `cloudbuild-manual-setup/01.setup` provisions dependencies required to setup Cloud Build for Playground: +- Required API services +- Cloud Build service account +- IAM permissions for Cloud Build service account + +#### Execute the module + +```console +# Create new configuration to authenticate to GCP Project +gcloud init + +# Acquire new user credentials to use for Application Default Credentials +gcloud auth application-default login + +# Navigate to cloudbuild-manual-setup/01.setup folder +cd 01.setup + +# Run terraform scripts +terraform init +terraform plan +terraform apply + +``` +### 2. Connect GitHub repository and Cloud Build +Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github) to connect GitHub repository and Cloud Build. + +### 3. Setup the Google Cloud Build triggers + +The `cloudbuild-manual-setup/02.builders` provisions: +- Cloud Build trigger to build and deploy Beam Playground. + +#### Execute the module + +Set required variables for GitHub repository owner, name and branch. + +``` +GITHUB_REPOSITORY_OWNER= +GITHUB_REPOSITORY_NAME= +GITHUB_REPOSITORY_BRANCH= +``` +Run the following commands to execute the module. Terraform will ask your permission before provisioning resources. If you agree with terraform provisioning resources, type `yes` to proceed. + +``` +# Navigate to cloudbuild-manual-setup/02.builders folder from cloudbuild-manual-setup/01.setup +cd ../02.builders +# Run terraform scripts +terraform init +terraform plan -var="github_repository_owner=$GITHUB_REPOSITORY_OWNER" -var="github_repository_name=$GITHUB_REPOSITORY_NAME" -var="github_repository_branch=$GITHUB_REPOSITORY_BRANCH" +terraform apply -var="github_repository_owner=$GITHUB_REPOSITORY_OWNER" -var="github_repository_name=$GITHUB_REPOSITORY_NAME" -var="github_repository_branch=$GITHUB_REPOSITORY_BRANCH" +``` + +After completing these steps, 2 Cloud Build triggers will be created in your GCP project that could be used to build and deploy Beam Playground. + +### Run Cloud Build Triggers + +## Requirement: + +To begin deploying Playground using Cloud Build triggers you must first execute steps described in +[Prepare deployment configuration](https://github.com/apache/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#prepare-deployment-configuration). + +#### After that, two triggers will have to be executed in sequence. + +## Trigger 1 - Playground-infrastructure-trigger: + +To run FIRST trigger, +navigate to Cloud Build page of your GCP project and click `RUN` for **Playground-infrastructure-trigger**. + +Once Playground infrastructure has been deployed, please navigate to +[Playground deployment README](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#deploy-playground-infrastructure) and execute step #2: + +`Add following DNS A records for the discovered static IP address` + +## Trigger 2 - Playground-to-gke-trigger: + +Once Playground infrastructure deployed, you could now deploy Playground to GKE. +To run SECOND trigger, +navigate to Cloud Build page of your GCP project and click `RUN` for **Playground-to-gke-trigger**. + +Once Playground has been deployed to GKE, please navigate to [Validation](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file From 19e24839ed67aacda9b2b1b0a199ae64e934777e Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 14 Nov 2022 01:12:08 +0500 Subject: [PATCH 003/269] Update README.md --- .../infrastructure/cloudbuild-manual-setup/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 6cfeec24aaad..2e1e988ad8a1 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -25,7 +25,7 @@ This directory organizes Infrastructure-as-Code to provision dependent resources - Enabled [Cloud Build API](https://cloud.google.com/apis/docs/getting-started#enabling_apis) - [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) - An existing Google Cloud Storage Bucket to save Terraform state - `state_bucket` -- And 2nd existing GCP Bucket to save Cloud build logs +- An existing GCP Bucket to save Cloud build logs - DNS name for your Playground deployment instance - [Terraform](https://www.terraform.io/) - [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally @@ -43,7 +43,7 @@ The `cloudbuild-manual-setup/01.setup` provisions dependencies required to setup #### Execute the module -```console +```consolec # Create new configuration to authenticate to GCP Project gcloud init From c077c49c3da17f1ff6af7738b7e728f198acb678 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 14 Nov 2022 16:20:31 +0500 Subject: [PATCH 004/269] Updates related to apache #24007, #24006 --- .../cloudbuild/cloudbuild_pg_infra.yaml | 12 ++++++------ .../cloudbuild/cloudbuild_pg_to_gke.yaml | 18 +++++++++--------- .../cloudbuild-manual-setup/README.md | 6 ++++-- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 13290a9a0322..244754fc1ece 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -32,15 +32,15 @@ steps: args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] substitutions: - # The name of the Google Cloud Artifact Registry - _ARTIFACT_REGISTRY_REPOSITORY_ID: "playground-repository" + # The name of your Google Cloud Artifact Registry + _ARTIFACT_REGISTRY_REPOSITORY_ID: "CHANGE_ME" # The name of your environment in beam/playground/terraform/environment/ - _ENVIRONMENT_NAME: "test" + _ENVIRONMENT_NAME: "CHANGE_ME" # The name of your DNS for Playground - _DNS_NAME: "dev-playground.online" + _DNS_NAME: "CHANGE_ME" -# Please change to your logs bucket -logsBucket: 'gs://pg-cloudbuild-logging-bucket' +# The name of your logs bucket +logsBucket: 'gs://CHANGE_ME' # Pushing built image to Artifact repository images: diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index e4b2e0471341..1d9d4fccdf6c 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -35,19 +35,19 @@ steps: args: [ '-c', './gradlew playground:terraform:gkebackend -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Pdns-name="$_DNS_NAME"' ] substitutions: - # The name of the Google Cloud Artifact Registry - _ARTIFACT_REGISTRY_REPOSITORY_ID: "playground-repository" + # The name of your Google Cloud Artifact Registry + _ARTIFACT_REGISTRY_REPOSITORY_ID: "CHANGE_ME" # The name of your environment in beam/playground/terraform/environment/ - _ENVIRONMENT_NAME: "test" + _ENVIRONMENT_NAME: "CHANGE_ME" # The name of your DNS for Playground - _DNS_NAME: "dev-playground.online" - # Docker image tag name - _TAG: "ruslan" + _DNS_NAME: "CHANGE_ME" + # The name of your Docker tag + _TAG: "CHANGE_ME" # The name of your GKE cluster (from terraform.tfvars) - _GKE_NAME: "playground-backend" + _GKE_NAME: "CHANGE_ME" -# Please change to your logs bucket -logsBucket: 'gs://pg-cloudbuild-logging-bucket' +# The name of your logs bucket +logsBucket: 'gs://CHANGE_ME' # This option is for writing logs to GCS bucket options: diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 2e1e988ad8a1..355fb605a447 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -101,7 +101,8 @@ To begin deploying Playground using Cloud Build triggers you must first execute ## Trigger 1 - Playground-infrastructure-trigger: To run FIRST trigger, -navigate to Cloud Build page of your GCP project and click `RUN` for **Playground-infrastructure-trigger**. +You have to change variables in SUBSTITUTIONS block in cloud build config file (playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml), +then navigate to Cloud Build page of your GCP project and click `RUN` for **Playground-infrastructure-trigger**. Once Playground infrastructure has been deployed, please navigate to [Playground deployment README](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#deploy-playground-infrastructure) and execute step #2: @@ -112,6 +113,7 @@ Once Playground infrastructure has been deployed, please navigate to Once Playground infrastructure deployed, you could now deploy Playground to GKE. To run SECOND trigger, -navigate to Cloud Build page of your GCP project and click `RUN` for **Playground-to-gke-trigger**. +You have to change variables in SUBSTITUTIONS block in cloud build config file (playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml), +then navigate to Cloud Build page of your GCP project and click `RUN` for **Playground-to-gke-trigger**. Once Playground has been deployed to GKE, please navigate to [Validation](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file From 3e45f35f76f305edd2020458d164ab3dbccbe066 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 15 Nov 2022 20:08:03 +0500 Subject: [PATCH 005/269] Updates to comments in apache #24144 --- .../01.setup/variables.tf | 2 - .../02.builders/variables.tf | 2 - .../cloudbuild-manual-setup/README.md | 123 ++++++++++-------- 3 files changed, 70 insertions(+), 57 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf index f38eeaf8358c..55e384f1feff 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf @@ -23,11 +23,9 @@ variable "project" { variable "region" { type = string description = "The Google Cloud Platform (GCP) region in which to provision resources" - default = "us-central1" } variable "cloudbuild_service_account_id" { type = string description = "The ID of the cloud build service account responsible for provisioning Google Cloud resources" - default = "terraform-cloudbuild" } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf index c48318c1f99a..4fde3aebdc76 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf @@ -23,7 +23,6 @@ variable "project" { variable "region" { type = string description = "The Google Cloud Platform (GCP) region in which to provision resources" - default = "us-central1" } variable "github_repository_name" { @@ -56,5 +55,4 @@ variable "gke_trigger_id" { variable "cloudbuild_service_account_id" { type = string description = "The ID of the cloud build service account responsible for provisioning Google Cloud resources" - default = "terraform-cloudbuild" } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 355fb605a447..198e46a05a9a 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -21,99 +21,116 @@ This directory organizes Infrastructure-as-Code to provision dependent resources ## Requirements: -- [GCP project](https://cloud.google.com/) -- Enabled [Cloud Build API](https://cloud.google.com/apis/docs/getting-started#enabling_apis) +- [GCP project](https://cloud.google.com/resource-manager/docs/creating-managing-projects) +- [GCP User account](https://cloud.google.com/appengine/docs/standard/access-control?tab=python) _(Note: You will find the instruction "How to create User account" for your new project)_
+ Ensure that the account has at least following privileges: + - Service Account Admin + - Storage Admin + - Service Usage Admin + - Cloud Build Editor - [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) -- An existing Google Cloud Storage Bucket to save Terraform state - `state_bucket` -- An existing GCP Bucket to save Cloud build logs +- Two GCP Buckets: + 1. An existing Google Cloud Storage Bucket to save Terraform state - `state_bucket` + 2. An existing GCP Bucket to save Cloud build logs - `cloudbuild_Logs_bucket` - DNS name for your Playground deployment instance - [Terraform](https://www.terraform.io/) - [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally -## Usage - -The folders are named according to the required order of execution. - -### 1. Setup the Google Cloud Build for your GCP project - -The `cloudbuild-manual-setup/01.setup` provisions dependencies required to setup Cloud Build for Playground: +## 1. Setup the Google Cloud Build for your GCP project + +The `playground/infrastructure/cloudbuild-manual-setup/01.setup` provisions dependencies required to setup Cloud Build for Playground: - Required API services - Cloud Build service account - IAM permissions for Cloud Build service account -#### Execute the module +#### To execute the module: + +1. Create configuration file terraform.tfvars file by path: `playground/terraform/cloudbuild-manual-setup/01.setup/terraform.tfvars` +and provide values as per example below: -```consolec -# Create new configuration to authenticate to GCP Project +``` +project = "sandbox-playground-001" # Your Project ID +region = "us-central1" # Your GCP region name where resources will be provisioned +cloudbuild_service_account_id = "terraform-cloudbuild" # Name of SA to be used by Cloud Build to deploy Playground +``` + +2. Run commands: + +```console +# Creates new authentication configuration to GCP Project +# Using user account in Requirements gcloud init -# Acquire new user credentials to use for Application Default Credentials +# Acquires new user account credentials to use for Application Default Credentials gcloud auth application-default login -# Navigate to cloudbuild-manual-setup/01.setup folder -cd 01.setup +# Navigate to ../01.setup folder +cd beam/playground/terraform/cloudbuild-manual-setup/01.setup # Run terraform scripts -terraform init -terraform plan -terraform apply +terraform init -backend-config="bucket="YOUR_BUCKET_FOR_TF_STATE" +terraform plan -var-file="../terraform.tfvars" +terraform apply -var-file="../terraform.tfvars" ``` -### 2. Connect GitHub repository and Cloud Build + +## 2. Connect GitHub repository and Cloud Build Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github) to connect GitHub repository and Cloud Build. -### 3. Setup the Google Cloud Build triggers +## 3. Setup the Google Cloud Build triggers -The `cloudbuild-manual-setup/02.builders` provisions: +The `playground/infrastructure/cloudbuild-manual-setup/02.builders` provisions: - Cloud Build trigger to build and deploy Beam Playground. #### Execute the module -Set required variables for GitHub repository owner, name and branch. +1. Append to `terraform.tfvars` file by path: `playground/terraform/cloudbuild-manual-setup/01.setup/terraform.tfvars` +next variables: ``` -GITHUB_REPOSITORY_OWNER= -GITHUB_REPOSITORY_NAME= -GITHUB_REPOSITORY_BRANCH= +github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' +github_repository_owner = "ruslan-ikhsan" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. +github_repository_branch = "cloudbuild+playground" # The name of GitHub repo branch ``` -Run the following commands to execute the module. Terraform will ask your permission before provisioning resources. If you agree with terraform provisioning resources, type `yes` to proceed. + +Final version of `terraform.tfvars` will look like: ``` -# Navigate to cloudbuild-manual-setup/02.builders folder from cloudbuild-manual-setup/01.setup -cd ../02.builders -# Run terraform scripts -terraform init -terraform plan -var="github_repository_owner=$GITHUB_REPOSITORY_OWNER" -var="github_repository_name=$GITHUB_REPOSITORY_NAME" -var="github_repository_branch=$GITHUB_REPOSITORY_BRANCH" -terraform apply -var="github_repository_owner=$GITHUB_REPOSITORY_OWNER" -var="github_repository_name=$GITHUB_REPOSITORY_NAME" -var="github_repository_branch=$GITHUB_REPOSITORY_BRANCH" +project = "sandbox-playground-001" # Your Project ID +region = "us-central1" # Your GCP region name where resources will be provisioned +cloudbuild_service_account_id = "terraform-cloudbuild" # Name of SA to be used by Cloud Build to deploy Playground +github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' +github_repository_owner = "ruslan-ikhsan" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. +github_repository_branch = "cloudbuild+playground" # The name of GitHub repo branch ``` -After completing these steps, 2 Cloud Build triggers will be created in your GCP project that could be used to build and deploy Beam Playground. - -### Run Cloud Build Triggers +2. Run: -## Requirement: - -To begin deploying Playground using Cloud Build triggers you must first execute steps described in -[Prepare deployment configuration](https://github.com/apache/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#prepare-deployment-configuration). +``` +# Navigate to ../02.builders folder +cd beam/playground/terraform/cloudbuild-manual-setup/02.builders +# Run terraform scripts +terraform init -backend-config="bucket="YOUR_BUCKET_FOR_TF_STATE" +terraform plan -var-file="terraform.tfvars" +terraform apply -var-file="terraform.tfvars" +``` -#### After that, two triggers will have to be executed in sequence. +## 4. Running First Cloud Build Trigger: "Playground-infrastructure-trigger" -## Trigger 1 - Playground-infrastructure-trigger: +1. Perform steps described in [Prepare deployment configuration](https://github.com/apache/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#prepare-deployment-configuration). -To run FIRST trigger, -You have to change variables in SUBSTITUTIONS block in cloud build config file (playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml), -then navigate to Cloud Build page of your GCP project and click `RUN` for **Playground-infrastructure-trigger**. +2. Change the variables in `SUBSTITUTIONS` block in first cloud build config file `playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml`. +3. Navigate to GCP Console. Open Cloud Build page -> Triggers. Sort by relevant region and click `RUN` for `Playground-infrastructure-trigger`. -Once Playground infrastructure has been deployed, please navigate to +4. Once Playground infrastructure has been deployed, please navigate to [Playground deployment README](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#deploy-playground-infrastructure) and execute step #2: - `Add following DNS A records for the discovered static IP address` -## Trigger 2 - Playground-to-gke-trigger: +## 5. Running Second Cloud Build Trigger: "Playground-to-gke-trigger" + +1. Change variables in `SUBSTITUTIONS` block in second cloud build config file `playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml` +2. Navigate to GCP Console. Open Cloud Build page -> Triggers. Sort by relevant region and click `RUN` for `Playground-to-gke-trigger`. -Once Playground infrastructure deployed, you could now deploy Playground to GKE. -To run SECOND trigger, -You have to change variables in SUBSTITUTIONS block in cloud build config file (playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml), -then navigate to Cloud Build page of your GCP project and click `RUN` for **Playground-to-gke-trigger**. +## 6. Validation Once Playground has been deployed to GKE, please navigate to [Validation](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file From 337174cefe528a3192b1c3fd89e484c7215bb2dd Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 15 Nov 2022 20:21:04 +0500 Subject: [PATCH 006/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 198e46a05a9a..c8da0708d2f2 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -131,6 +131,6 @@ terraform apply -var-file="terraform.tfvars" 1. Change variables in `SUBSTITUTIONS` block in second cloud build config file `playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml` 2. Navigate to GCP Console. Open Cloud Build page -> Triggers. Sort by relevant region and click `RUN` for `Playground-to-gke-trigger`. -## 6. Validation +## 6. Validation Once Playground has been deployed to GKE, please navigate to [Validation](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file From 0c3fa3690502489efa334f79ae89702c3262d256 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 15 Nov 2022 20:38:56 +0500 Subject: [PATCH 007/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index c8da0708d2f2..ba657b17d64c 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -57,7 +57,7 @@ cloudbuild_service_account_id = "terraform-cloudbuild" # Name of SA to be u 2. Run commands: ```console -# Creates new authentication configuration to GCP Project +# Creates new authentication configuration to GCP Project # Using user account in Requirements gcloud init From 5e8fc11db9003ab4835a42a0d6a51fb3667706b0 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 15 Nov 2022 21:08:32 +0500 Subject: [PATCH 008/269] Update README.md --- .../infrastructure/cloudbuild-manual-setup/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index ba657b17d64c..5c9971c8b6e0 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -37,7 +37,7 @@ This directory organizes Infrastructure-as-Code to provision dependent resources - [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally ## 1. Setup the Google Cloud Build for your GCP project - + The `playground/infrastructure/cloudbuild-manual-setup/01.setup` provisions dependencies required to setup Cloud Build for Playground: - Required API services - Cloud Build service account @@ -58,7 +58,7 @@ cloudbuild_service_account_id = "terraform-cloudbuild" # Name of SA to be u ```console # Creates new authentication configuration to GCP Project -# Using user account in Requirements +# Using user account in Requirements gcloud init # Acquires new user account credentials to use for Application Default Credentials From 015cb2f021b886b09891b54f45ba4317410549cd Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 15 Nov 2022 22:25:25 +0500 Subject: [PATCH 009/269] Optimizing cloud build yaml files --- .../cloudbuild/cloudbuild_pg_infra.yaml | 14 ++++---- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 35 +++++++++++-------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 244754fc1ece..4661bf65fa4b 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -20,31 +20,33 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: - 'build' - - '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/builder:worker' + - '--tag=${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' - '-f' - 'playground/infrastructure/cloudbuild/Dockerfile-base-image-worker' - '.' # This step uses Docker container from image in previous step to run gradle task for Playground Infrastructure deployment - - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/builder:worker' + - name: '$LOCATION-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' entrypoint: "/bin/bash" # Please set variables $_ENVIRONMENT_NAME and $_DNS_NAME in "substitutions" stage below - args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] + args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="${_ENVIRONMENT_NAME}" -Pdns-name="${_DNS_NAME}"'] substitutions: # The name of your Google Cloud Artifact Registry - _ARTIFACT_REGISTRY_REPOSITORY_ID: "CHANGE_ME" + _ARTIFACT_REGISTRY_REPO: "CHANGE_ME" # The name of your environment in beam/playground/terraform/environment/ _ENVIRONMENT_NAME: "CHANGE_ME" # The name of your DNS for Playground _DNS_NAME: "CHANGE_ME" + # The name of the GCP logs bucket + _LOGS_BUCKET_NAME: "CHANGE_ME" # The name of your logs bucket -logsBucket: 'gs://CHANGE_ME' +logsBucket: 'gs://${_LOGS_BUCKET_NAME}' # Pushing built image to Artifact repository images: - - '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/builder:worker' + - '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' # This option is for writing logs to GCS bucket options: diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 1d9d4fccdf6c..a15130d07e6f 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -18,25 +18,30 @@ steps: # This step uses Docker container from our image to authenticate us in GKE # Please set variable $_GKE_NAME in "substitutions" stage below - - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/builder:worker' - entrypoint: "gcloud" - args: - [ - 'container', - 'clusters', - 'get-credentials', - '$_GKE_NAME', - '--region=$LOCATION-a' - ] +# - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPO/builder:worker' +# entrypoint: "gcloud" +# args: +# [ +# 'container', +# 'clusters', +# 'get-credentials', +# '${_GKE_NAME}', +# '--region=${LOCATION}-a' +# ] # This step uses Docker container from image in previous step to run gradle task for Playground to GKE deployment - - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/builder:worker' + - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' entrypoint: "/bin/bash" - args: [ '-c', './gradlew playground:terraform:gkebackend -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Pdns-name="$_DNS_NAME"' ] + args: + - '-c' + - | + ./gradlew playground:terraform:prepareConfig -Pdns-name="${_DNS_NAME}" && + ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="${_DOCKER_REPOSITORY_ROOT}" \ + -Pproject_environment="{$_ENVIRONMENT_NAME}" -Pdocker-tag="${_TAG}" -Pdns-name="${_DNS_NAME}"' substitutions: # The name of your Google Cloud Artifact Registry - _ARTIFACT_REGISTRY_REPOSITORY_ID: "CHANGE_ME" + _ARTIFACT_REGISTRY_REPO: "CHANGE_ME" # The name of your environment in beam/playground/terraform/environment/ _ENVIRONMENT_NAME: "CHANGE_ME" # The name of your DNS for Playground @@ -45,9 +50,11 @@ substitutions: _TAG: "CHANGE_ME" # The name of your GKE cluster (from terraform.tfvars) _GKE_NAME: "CHANGE_ME" + # The name of the GCP logs bucket + _LOGS_BUCKET_NAME: "CHANGE_ME" # The name of your logs bucket -logsBucket: 'gs://CHANGE_ME' +logsBucket: 'gs://${_LOGS_BUCKET_NAME}' # This option is for writing logs to GCS bucket options: From a52aea02f2bb4d87bbb3122fdbc40c3ae430c79a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 16 Nov 2022 19:15:21 +0500 Subject: [PATCH 010/269] Updates related to apache #24007, #24006 --- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 13 ---------- .../cloudbuild-manual-setup/README.md | 24 +++++++++++++------ 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index a15130d07e6f..9cec92f173c5 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -16,19 +16,6 @@ # under the License. steps: - # This step uses Docker container from our image to authenticate us in GKE - # Please set variable $_GKE_NAME in "substitutions" stage below -# - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPO/builder:worker' -# entrypoint: "gcloud" -# args: -# [ -# 'container', -# 'clusters', -# 'get-credentials', -# '${_GKE_NAME}', -# '--region=${LOCATION}-a' -# ] - # This step uses Docker container from image in previous step to run gradle task for Playground to GKE deployment - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' entrypoint: "/bin/bash" diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 5c9971c8b6e0..7fda0145372b 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -119,18 +119,28 @@ terraform apply -var-file="terraform.tfvars" 1. Perform steps described in [Prepare deployment configuration](https://github.com/apache/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#prepare-deployment-configuration). -2. Change the variables in `SUBSTITUTIONS` block in first cloud build config file `playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml`. -3. Navigate to GCP Console. Open Cloud Build page -> Triggers. Sort by relevant region and click `RUN` for `Playground-infrastructure-trigger`. - -4. Once Playground infrastructure has been deployed, please navigate to +2. Navigate to GCP Console. Open Cloud Build page -> Triggers. +3. Choose your Region. +4. Open `Playground-infrastructure-trigger`. +5. Scroll down to `Advanced` - `Substitutions variables`. +6. Click on `+ ADD VARIABLE` +7. Assign values for next variables: + - `_ARTIFACT_REGISTRY_REPO` *#Your GCP artifact repo name* + - `_ENVIRONMENT_NAME` *#Your env name in beam/playground/terraform/environment/* + - `_DNS_NAME` *#Your DNS for Playground* + - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name* + +8. Once Playground infrastructure has been deployed, please navigate to [Playground deployment README](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#deploy-playground-infrastructure) and execute step #2: `Add following DNS A records for the discovered static IP address` ## 5. Running Second Cloud Build Trigger: "Playground-to-gke-trigger" -1. Change variables in `SUBSTITUTIONS` block in second cloud build config file `playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml` -2. Navigate to GCP Console. Open Cloud Build page -> Triggers. Sort by relevant region and click `RUN` for `Playground-to-gke-trigger`. +1. Navigate to GCP Console. Open Cloud Build page -> Triggers. +2. Provide values for next variables in `Advanced` - `Substitutions variables`: +3. +3. Sort by relevant region and click `RUN` for `Playground-to-gke-trigger`. ## 6. Validation -Once Playground has been deployed to GKE, please navigate to [Validation](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file +Once Playground has been deployed to GKE, please navigate to [Validation](ht tps://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file From 2c9b70e67b45d3b485d4735e209a921d9b43f303 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 16 Nov 2022 19:30:57 +0500 Subject: [PATCH 011/269] Updates related to apache #24006, #24007 --- .../cloudbuild/cloudbuild_pg_infra.yaml | 22 ++++++++-------- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 21 ++++++++-------- .../cloudbuild-manual-setup/README.md | 25 ++++++++++++++----- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 4661bf65fa4b..2d3629511127 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -16,7 +16,7 @@ # under the License. steps: - # This step builds the image from Dockerfile that will be used as Docker machine for Beam Playground deployment + # This stage builds the image from Dockerfile that will be used as Docker environment to run Beam Playground deployment - name: 'gcr.io/cloud-builders/docker' args: - 'build' @@ -25,30 +25,30 @@ steps: - 'playground/infrastructure/cloudbuild/Dockerfile-base-image-worker' - '.' - # This step uses Docker container from image in previous step to run gradle task for Playground Infrastructure deployment + # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - name: '$LOCATION-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' entrypoint: "/bin/bash" - # Please set variables $_ENVIRONMENT_NAME and $_DNS_NAME in "substitutions" stage below args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="${_ENVIRONMENT_NAME}" -Pdns-name="${_DNS_NAME}"'] +# Substitutions can be changed in GCP Cloud Build Triggers UI substitutions: - # The name of your Google Cloud Artifact Registry - _ARTIFACT_REGISTRY_REPO: "CHANGE_ME" + # The name of your GCP Artifact Registry repository + _ARTIFACT_REGISTRY_REPO: "playground-repository" # The name of your environment in beam/playground/terraform/environment/ - _ENVIRONMENT_NAME: "CHANGE_ME" + _ENVIRONMENT_NAME: "test" # The name of your DNS for Playground - _DNS_NAME: "CHANGE_ME" + _DNS_NAME: "dev-playground.online" # The name of the GCP logs bucket - _LOGS_BUCKET_NAME: "CHANGE_ME" + _LOGS_BUCKET_NAME: "pg-cloudbuild-logging-bucket" -# The name of your logs bucket +# The name of your GCP logs bucket logsBucket: 'gs://${_LOGS_BUCKET_NAME}' -# Pushing built image to Artifact repository +# Pushing built image to GCP Artifact Registry repository images: - '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' -# This option is for writing logs to GCS bucket +# This option enables writing logs to GCS bucket options: logging: GCS_ONLY diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 9cec92f173c5..2784e6db48bb 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -16,7 +16,7 @@ # under the License. steps: - # This step uses Docker container from image in previous step to run gradle task for Playground to GKE deployment + # This stage uses pre-built Docker container to run gradle task for Playground pre-config and deployment to GKE - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' entrypoint: "/bin/bash" args: @@ -26,24 +26,25 @@ steps: ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="${_DOCKER_REPOSITORY_ROOT}" \ -Pproject_environment="{$_ENVIRONMENT_NAME}" -Pdocker-tag="${_TAG}" -Pdns-name="${_DNS_NAME}"' +# Substitutions can be changed in GCP Cloud Build Triggers UI substitutions: - # The name of your Google Cloud Artifact Registry - _ARTIFACT_REGISTRY_REPO: "CHANGE_ME" + # The name of your Google Cloud Artifact Registry repository + _ARTIFACT_REGISTRY_REPO: "playground-repository" # The name of your environment in beam/playground/terraform/environment/ - _ENVIRONMENT_NAME: "CHANGE_ME" + _ENVIRONMENT_NAME: "test" # The name of your DNS for Playground - _DNS_NAME: "CHANGE_ME" + _DNS_NAME: "dev-playground.online" # The name of your Docker tag - _TAG: "CHANGE_ME" + _TAG: "tag-playground" # The name of your GKE cluster (from terraform.tfvars) - _GKE_NAME: "CHANGE_ME" + _GKE_NAME: "playground-cluster" # The name of the GCP logs bucket - _LOGS_BUCKET_NAME: "CHANGE_ME" + _LOGS_BUCKET_NAME: "pg-cloudbuild-logging-bucket" -# The name of your logs bucket +# The name of your GCP logs bucket logsBucket: 'gs://${_LOGS_BUCKET_NAME}' -# This option is for writing logs to GCS bucket +# This option enables writing logs to GCS bucket options: logging: GCS_ONLY diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 7fda0145372b..f6ff9bffb2c7 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -121,7 +121,7 @@ terraform apply -var-file="terraform.tfvars" 2. Navigate to GCP Console. Open Cloud Build page -> Triggers. 3. Choose your Region. -4. Open `Playground-infrastructure-trigger`. +4. Open Trigger: `Playground-infrastructure-trigger`. 5. Scroll down to `Advanced` - `Substitutions variables`. 6. Click on `+ ADD VARIABLE` 7. Assign values for next variables: @@ -129,17 +129,30 @@ terraform apply -var-file="terraform.tfvars" - `_ENVIRONMENT_NAME` *#Your env name in beam/playground/terraform/environment/* - `_DNS_NAME` *#Your DNS for Playground* - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name* +8. Click save. +9. Run the trigger `Playground-infrastructure-trigger`. -8. Once Playground infrastructure has been deployed, please navigate to +10. Once Playground infrastructure has been deployed, please navigate to [Playground deployment README](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#deploy-playground-infrastructure) and execute step #2: `Add following DNS A records for the discovered static IP address` ## 5. Running Second Cloud Build Trigger: "Playground-to-gke-trigger" -1. Navigate to GCP Console. Open Cloud Build page -> Triggers. -2. Provide values for next variables in `Advanced` - `Substitutions variables`: -3. -3. Sort by relevant region and click `RUN` for `Playground-to-gke-trigger`. +1. Navigate to GCP Console. Open Cloud Build page -> Triggers. +2. Choose your Region. +3. Open Trigger: `Playground-to-gke-trigger`. +4. Scroll down to `Advanced` - `Substitutions variables`. +5. Click on `+ ADD VARIABLE` +6. Assign values for next variables: + - `_ARTIFACT_REGISTRY_REPO` *#Your GCP artifact repo name* + - `_ENVIRONMENT_NAME` *#Your env name in beam/playground/terraform/environment/* + - `_DNS_NAME` *#Your DNS for Playground* + - `_TAG` *#Tag name for your Playground container images* + - `_GKE_NAME` *#Your GKE cluster name for Playground* + - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name* +7. Click save. +8. Run the trigger `Playground-to-gke-trigger`. + ## 6. Validation From 4a1d48e3220b9d70c8b4d046b851be4cd979e19f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 16 Nov 2022 19:32:08 +0500 Subject: [PATCH 012/269] Update README.md --- .../infrastructure/cloudbuild-manual-setup/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index f6ff9bffb2c7..da21206610fc 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -124,7 +124,7 @@ terraform apply -var-file="terraform.tfvars" 4. Open Trigger: `Playground-infrastructure-trigger`. 5. Scroll down to `Advanced` - `Substitutions variables`. 6. Click on `+ ADD VARIABLE` -7. Assign values for next variables: +7. Assign values for next variables: - `_ARTIFACT_REGISTRY_REPO` *#Your GCP artifact repo name* - `_ENVIRONMENT_NAME` *#Your env name in beam/playground/terraform/environment/* - `_DNS_NAME` *#Your DNS for Playground* @@ -156,4 +156,4 @@ terraform apply -var-file="terraform.tfvars" ## 6. Validation -Once Playground has been deployed to GKE, please navigate to [Validation](ht tps://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file +Once Playground has been deployed to GKE, please navigate to [Validation](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file From 58b9ef6af0e0f2b07aa499190d04290625345584 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 16 Nov 2022 22:13:55 +0500 Subject: [PATCH 013/269] Update cloudbuild_pg_to_gke.yaml --- .../infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 2784e6db48bb..4ad55b8b15d1 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -20,11 +20,11 @@ steps: - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' entrypoint: "/bin/bash" args: - - '-c' - - | - ./gradlew playground:terraform:prepareConfig -Pdns-name="${_DNS_NAME}" && + - '-c' + - | + ./gradlew playground:terraform:prepareConfig -Pdns-name="${_DNS_NAME}" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="${_DOCKER_REPOSITORY_ROOT}" \ - -Pproject_environment="{$_ENVIRONMENT_NAME}" -Pdocker-tag="${_TAG}" -Pdns-name="${_DNS_NAME}"' + -Pproject_environment="{$_ENVIRONMENT_NAME}" -Pdocker-tag="${_TAG}" -Pdns-name="${_DNS_NAME}"' # Substitutions can be changed in GCP Cloud Build Triggers UI substitutions: From cbcf3b4356cbf4d3197292d08b92e12832fcb3a8 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 17 Nov 2022 00:19:20 +0500 Subject: [PATCH 014/269] Added curly brackets for variables --- playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 2d3629511127..97ce93bc0653 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -20,13 +20,13 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: - 'build' - - '--tag=${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' + - '--tag=${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/playground-builder:worker' - '-f' - 'playground/infrastructure/cloudbuild/Dockerfile-base-image-worker' - '.' # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - - name: '$LOCATION-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' + - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' entrypoint: "/bin/bash" args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="${_ENVIRONMENT_NAME}" -Pdns-name="${_DNS_NAME}"'] From 111e3550d461009477fa30f6ccc5f7d93f813c7c Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 17 Nov 2022 00:21:20 +0500 Subject: [PATCH 015/269] Update cloudbuild_pg_to_gke.yaml --- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 4ad55b8b15d1..aa58788f87e9 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -20,11 +20,11 @@ steps: - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' entrypoint: "/bin/bash" args: - - '-c' - - | - ./gradlew playground:terraform:prepareConfig -Pdns-name="${_DNS_NAME}" - ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="${_DOCKER_REPOSITORY_ROOT}" \ - -Pproject_environment="{$_ENVIRONMENT_NAME}" -Pdocker-tag="${_TAG}" -Pdns-name="${_DNS_NAME}"' + [ + '-c', './gradlew playground:terraform:prepareConfig -Pdns-name="${_DNS_NAME}" && + ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="${_DOCKER_REPOSITORY_ROOT}" + -Pproject_environment="${_ENVIRONMENT_NAME}" -Pdocker-tag="${_TAG}" -Pdns-name="${_DNS_NAME}"' + ] # Substitutions can be changed in GCP Cloud Build Triggers UI substitutions: @@ -40,6 +40,8 @@ substitutions: _GKE_NAME: "playground-cluster" # The name of the GCP logs bucket _LOGS_BUCKET_NAME: "pg-cloudbuild-logging-bucket" + # The name of docker repository root (from outputs of first trigger result) + _DOCKER_REPOSITORY_ROOT: "" # The name of your GCP logs bucket logsBucket: 'gs://${_LOGS_BUCKET_NAME}' From 2002062e9ec3d97a1e02c367a6a33d4a91bf1d06 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 17 Nov 2022 15:10:59 +0500 Subject: [PATCH 016/269] Test updates for apache #24007 --- .../01.setup/bucket.tf | 24 ++++++++++++++++ .../01.setup/variables.tf | 5 ++++ .../cloudbuild-manual-setup/README.md | 28 +++++++++++-------- 3 files changed, 45 insertions(+), 12 deletions(-) create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf new file mode 100644 index 000000000000..41242600fda9 --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +resource "google_storage_bucket" "cloudbuild_logs_bucket" { + name = var.cloudbuild_logs_bucket_name + location = var.region + storage_class = "STANDARD" + + public_access_prevention = "enforced" +} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf index 55e384f1feff..3dde46d0fee9 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf @@ -28,4 +28,9 @@ variable "region" { variable "cloudbuild_service_account_id" { type = string description = "The ID of the cloud build service account responsible for provisioning Google Cloud resources" +} + +variable "cloudbuild_logs_bucket_name" { + type = string + description = "Globally unique ID of your logs bucket for Cloud Build" } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index da21206610fc..a706fa43d642 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -28,30 +28,33 @@ This directory organizes Infrastructure-as-Code to provision dependent resources - Storage Admin - Service Usage Admin - Cloud Build Editor + - Security Admin + Service Account User - [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) - Two GCP Buckets: - 1. An existing Google Cloud Storage Bucket to save Terraform state - `state_bucket` - 2. An existing GCP Bucket to save Cloud build logs - `cloudbuild_Logs_bucket` + 1. An existing GCP Bucket to save Terraform state - `state_bucket` - DNS name for your Playground deployment instance - [Terraform](https://www.terraform.io/) - [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally ## 1. Setup the Google Cloud Build for your GCP project -The `playground/infrastructure/cloudbuild-manual-setup/01.setup` provisions dependencies required to setup Cloud Build for Playground: +The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provisions dependencies required to setup Cloud Build for Playground: - Required API services - Cloud Build service account - IAM permissions for Cloud Build service account +- GCP Storage Bucket for Cloud Build logs #### To execute the module: -1. Create configuration file terraform.tfvars file by path: `playground/terraform/cloudbuild-manual-setup/01.setup/terraform.tfvars` +1. Create configuration file terraform.tfvars file by path: `playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars` and provide values as per example below: ``` project = "sandbox-playground-001" # Your Project ID region = "us-central1" # Your GCP region name where resources will be provisioned cloudbuild_service_account_id = "terraform-cloudbuild" # Name of SA to be used by Cloud Build to deploy Playground +cloudbuild_logs_bucket_name = "cb_logs_bucket" # The name of GCP bucket for Cloud Build logs ``` 2. Run commands: @@ -65,10 +68,10 @@ gcloud init gcloud auth application-default login # Navigate to ../01.setup folder -cd beam/playground/terraform/cloudbuild-manual-setup/01.setup +cd playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/ # Run terraform scripts -terraform init -backend-config="bucket="YOUR_BUCKET_FOR_TF_STATE" +terraform init -backend-config="bucket=YOUR_BUCKET_FOR_TF_STATE" terraform plan -var-file="../terraform.tfvars" terraform apply -var-file="../terraform.tfvars" @@ -79,7 +82,7 @@ Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/auto ## 3. Setup the Google Cloud Build triggers -The `playground/infrastructure/cloudbuild-manual-setup/02.builders` provisions: +The `playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders` provisions: - Cloud Build trigger to build and deploy Beam Playground. #### Execute the module @@ -89,7 +92,7 @@ next variables: ``` github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' -github_repository_owner = "ruslan-ikhsan" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. +github_repository_owner = "owner" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. github_repository_branch = "cloudbuild+playground" # The name of GitHub repo branch ``` @@ -99,8 +102,9 @@ Final version of `terraform.tfvars` will look like: project = "sandbox-playground-001" # Your Project ID region = "us-central1" # Your GCP region name where resources will be provisioned cloudbuild_service_account_id = "terraform-cloudbuild" # Name of SA to be used by Cloud Build to deploy Playground +cloudbuild_logs_bucket_name = "cb_logs_bucket" # The name of GCP bucket for Cloud Build logs github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' -github_repository_owner = "ruslan-ikhsan" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. +github_repository_owner = "owner" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. github_repository_branch = "cloudbuild+playground" # The name of GitHub repo branch ``` @@ -110,9 +114,9 @@ github_repository_branch = "cloudbuild+playground" # The name of GitHub # Navigate to ../02.builders folder cd beam/playground/terraform/cloudbuild-manual-setup/02.builders # Run terraform scripts -terraform init -backend-config="bucket="YOUR_BUCKET_FOR_TF_STATE" -terraform plan -var-file="terraform.tfvars" -terraform apply -var-file="terraform.tfvars" +terraform init -backend-config="bucket=YOUR_BUCKET_FOR_TF_STATE" +terraform plan -var-file="../terraform.tfvars" +terraform apply -var-file="../terraform.tfvars" ``` ## 4. Running First Cloud Build Trigger: "Playground-infrastructure-trigger" From 1e6cbf1798b90ca1a1da006ce650d76fa26f5f0c Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 17 Nov 2022 15:12:46 +0500 Subject: [PATCH 017/269] Create terraform.tfvars --- .../cloudbuild-manual-setup/terraform.tfvars | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars b/playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars new file mode 100644 index 000000000000..a0d3115685be --- /dev/null +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +project = "sandbox-playground-001" # Your Project ID +region = "us-central1" # Your GCP region name where resources will be provisioned +cloudbuild_service_account_id = "new-terraform-cloudbuild" # The Name of SA to be used by Cloud Build to deploy Playground +cloudbuild_logs_bucket_name = "cb-logs-bucket" # The name of GCP bucket for Cloud Build logs +github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' +github_repository_owner = "ruslan-ikhsan" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. +github_repository_branch = "cloudbuild+playground" # The name of GitHub repo branch \ No newline at end of file From 085b32447a1f0a74846c9b1241a39eacca00104e Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 17 Nov 2022 15:56:24 +0500 Subject: [PATCH 018/269] Latest updates related to PR apache #24144 --- .gitignore | 3 +- .../cloudbuild/cloudbuild_pg_infra.yaml | 8 ++--- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 4 +-- .../01.setup/bucket.tf | 2 -- .../cloudbuild-manual-setup/README.md | 34 +++++++++---------- .../cloudbuild-manual-setup/terraform.tfvars | 14 ++++---- 6 files changed, 29 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 62f5a20d45e7..6b9a09e67570 100644 --- a/.gitignore +++ b/.gitignore @@ -136,4 +136,5 @@ website/www/yarn-error.log **/*.tfvars # Ignore Katas auto-generated files -**/*-remote-info.yaml \ No newline at end of file +**/*-remote-info.yaml +playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 97ce93bc0653..1534339613ba 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -20,20 +20,18 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: - 'build' - - '--tag=${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/playground-builder:worker' + - '--tag=${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/playground-builder:worker' - '-f' - 'playground/infrastructure/cloudbuild/Dockerfile-base-image-worker' - '.' # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' + - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:worker' entrypoint: "/bin/bash" args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="${_ENVIRONMENT_NAME}" -Pdns-name="${_DNS_NAME}"'] # Substitutions can be changed in GCP Cloud Build Triggers UI substitutions: - # The name of your GCP Artifact Registry repository - _ARTIFACT_REGISTRY_REPO: "playground-repository" # The name of your environment in beam/playground/terraform/environment/ _ENVIRONMENT_NAME: "test" # The name of your DNS for Playground @@ -46,7 +44,7 @@ logsBucket: 'gs://${_LOGS_BUCKET_NAME}' # Pushing built image to GCP Artifact Registry repository images: - - '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' + - '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:worker' # This option enables writing logs to GCS bucket options: diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index aa58788f87e9..82079a81aef4 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -17,7 +17,7 @@ steps: # This stage uses pre-built Docker container to run gradle task for Playground pre-config and deployment to GKE - - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/builder:worker' + - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:worker' entrypoint: "/bin/bash" args: [ @@ -28,8 +28,6 @@ steps: # Substitutions can be changed in GCP Cloud Build Triggers UI substitutions: - # The name of your Google Cloud Artifact Registry repository - _ARTIFACT_REGISTRY_REPO: "playground-repository" # The name of your environment in beam/playground/terraform/environment/ _ENVIRONMENT_NAME: "test" # The name of your DNS for Playground diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf index 41242600fda9..361f4ac9117b 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf @@ -19,6 +19,4 @@ resource "google_storage_bucket" "cloudbuild_logs_bucket" { name = var.cloudbuild_logs_bucket_name location = var.region storage_class = "STANDARD" - - public_access_prevention = "enforced" } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index a706fa43d642..af2e54757df8 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -91,28 +91,28 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders` pr next variables: ``` -github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' -github_repository_owner = "owner" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. -github_repository_branch = "cloudbuild+playground" # The name of GitHub repo branch +github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' +github_repository_owner = "owner" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. +github_repository_branch = "cloudbuild+manualsetup+playground" # The name of GitHub repo branch ``` Final version of `terraform.tfvars` will look like: ``` -project = "sandbox-playground-001" # Your Project ID -region = "us-central1" # Your GCP region name where resources will be provisioned -cloudbuild_service_account_id = "terraform-cloudbuild" # Name of SA to be used by Cloud Build to deploy Playground -cloudbuild_logs_bucket_name = "cb_logs_bucket" # The name of GCP bucket for Cloud Build logs -github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' -github_repository_owner = "owner" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. -github_repository_branch = "cloudbuild+playground" # The name of GitHub repo branch +project = "sandbox-playground-001" # Your Project ID +region = "us-central1" # Your GCP region name where resources will be provisioned +cloudbuild_service_account_id = "terraform-cloudbuild" # Name of SA to be used by Cloud Build to deploy Playground +cloudbuild_logs_bucket_name = "cb_logs_bucket" # The name of GCP bucket for Cloud Build logs +github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' +github_repository_owner = "owner" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. +github_repository_branch = "cloudbuild+manualsetup+playground" # The name of GitHub repo branch ``` 2. Run: ``` -# Navigate to ../02.builders folder -cd beam/playground/terraform/cloudbuild-manual-setup/02.builders +# Navigate to playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders folder +cd ../02.builders # Run terraform scripts terraform init -backend-config="bucket=YOUR_BUCKET_FOR_TF_STATE" terraform plan -var-file="../terraform.tfvars" @@ -124,15 +124,14 @@ terraform apply -var-file="../terraform.tfvars" 1. Perform steps described in [Prepare deployment configuration](https://github.com/apache/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#prepare-deployment-configuration). 2. Navigate to GCP Console. Open Cloud Build page -> Triggers. -3. Choose your Region. +3. Choose region that was configured earlier in terraform.tfvars. 4. Open Trigger: `Playground-infrastructure-trigger`. 5. Scroll down to `Advanced` - `Substitutions variables`. 6. Click on `+ ADD VARIABLE` 7. Assign values for next variables: - - `_ARTIFACT_REGISTRY_REPO` *#Your GCP artifact repo name* - `_ENVIRONMENT_NAME` *#Your env name in beam/playground/terraform/environment/* - `_DNS_NAME` *#Your DNS for Playground* - - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name* + - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name (configured earlier in terraform.tfvars)* 8. Click save. 9. Run the trigger `Playground-infrastructure-trigger`. @@ -143,17 +142,16 @@ terraform apply -var-file="../terraform.tfvars" ## 5. Running Second Cloud Build Trigger: "Playground-to-gke-trigger" 1. Navigate to GCP Console. Open Cloud Build page -> Triggers. -2. Choose your Region. +2. Choose region that was configured earlier in terraform.tfvars. 3. Open Trigger: `Playground-to-gke-trigger`. 4. Scroll down to `Advanced` - `Substitutions variables`. 5. Click on `+ ADD VARIABLE` 6. Assign values for next variables: - - `_ARTIFACT_REGISTRY_REPO` *#Your GCP artifact repo name* - `_ENVIRONMENT_NAME` *#Your env name in beam/playground/terraform/environment/* - `_DNS_NAME` *#Your DNS for Playground* - `_TAG` *#Tag name for your Playground container images* - `_GKE_NAME` *#Your GKE cluster name for Playground* - - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name* + - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name (configured earlier in terraform.tfvars)* 7. Click save. 8. Run the trigger `Playground-to-gke-trigger`. diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars b/playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars index a0d3115685be..086dd70d452b 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -project = "sandbox-playground-001" # Your Project ID -region = "us-central1" # Your GCP region name where resources will be provisioned -cloudbuild_service_account_id = "new-terraform-cloudbuild" # The Name of SA to be used by Cloud Build to deploy Playground -cloudbuild_logs_bucket_name = "cb-logs-bucket" # The name of GCP bucket for Cloud Build logs -github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' -github_repository_owner = "ruslan-ikhsan" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. -github_repository_branch = "cloudbuild+playground" # The name of GitHub repo branch \ No newline at end of file +project = "sandbox-playground-001" # Your Project ID +region = "us-central1" # Your GCP region name where resources will be provisioned +cloudbuild_service_account_id = "new-terraform-cloudbuild" # The Name of SA to be used by Cloud Build to deploy Playground +cloudbuild_logs_bucket_name = "cb-logs-bucket" # The name of GCP bucket for Cloud Build logs +github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' +github_repository_owner = "akvelon" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. +github_repository_branch = "cloudbuild+manualsetup+playground" # The name of GitHub repo branch \ No newline at end of file From a74982d8220d8a961746f8967d1a76f520cb1189 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Thu, 17 Nov 2022 15:56:55 +0500 Subject: [PATCH 019/269] Delete terraform.tfvars --- .../cloudbuild-manual-setup/terraform.tfvars | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars b/playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars deleted file mode 100644 index 086dd70d452b..000000000000 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -project = "sandbox-playground-001" # Your Project ID -region = "us-central1" # Your GCP region name where resources will be provisioned -cloudbuild_service_account_id = "new-terraform-cloudbuild" # The Name of SA to be used by Cloud Build to deploy Playground -cloudbuild_logs_bucket_name = "cb-logs-bucket" # The name of GCP bucket for Cloud Build logs -github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' -github_repository_owner = "akvelon" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. -github_repository_branch = "cloudbuild+manualsetup+playground" # The name of GitHub repo branch \ No newline at end of file From c43c8bc2cf21fac589a5a0695e052a4922d2b206 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 17 Nov 2022 16:05:41 +0500 Subject: [PATCH 020/269] Update README.md --- .../cloudbuild-manual-setup/README.md | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index af2e54757df8..9d189a503cda 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -29,10 +29,9 @@ This directory organizes Infrastructure-as-Code to provision dependent resources - Service Usage Admin - Cloud Build Editor - Security Admin - Service Account User + - Service Account User - [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) -- Two GCP Buckets: - 1. An existing GCP Bucket to save Terraform state - `state_bucket` +- An existing GCP Bucket to save Terraform state - `state_bucket` - DNS name for your Playground deployment instance - [Terraform](https://www.terraform.io/) - [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally @@ -47,7 +46,7 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi #### To execute the module: -1. Create configuration file terraform.tfvars file by path: `playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars` +1. On your local environment create configuration file terraform.tfvars file by path: `playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars` and provide values as per example below: ``` @@ -60,18 +59,18 @@ cloudbuild_logs_bucket_name = "cb_logs_bucket" # The name of GCP bu 2. Run commands: ```console -# Creates new authentication configuration to GCP Project -# Using user account in Requirements +# Creates new authentication configuration to GCP Project with user account created in Requirements gcloud init # Acquires new user account credentials to use for Application Default Credentials gcloud auth application-default login -# Navigate to ../01.setup folder +# Navigate to 01.setup folder cd playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/ # Run terraform scripts -terraform init -backend-config="bucket=YOUR_BUCKET_FOR_TF_STATE" +# Where "bucket=" set your bucket name for TF state +terraform init -backend-config="bucket=playground-state-bucket" terraform plan -var-file="../terraform.tfvars" terraform apply -var-file="../terraform.tfvars" @@ -83,7 +82,7 @@ Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/auto ## 3. Setup the Google Cloud Build triggers The `playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders` provisions: -- Cloud Build trigger to build and deploy Beam Playground. +- Cloud Build triggers to build and deploy Beam Playground. #### Execute the module @@ -114,7 +113,8 @@ github_repository_branch = "cloudbuild+manualsetup+playground" # The na # Navigate to playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders folder cd ../02.builders # Run terraform scripts -terraform init -backend-config="bucket=YOUR_BUCKET_FOR_TF_STATE" +# Where "bucket=" set your bucket name for TF state +terraform init -backend-config="bucket=playground-state-bucket" terraform plan -var-file="../terraform.tfvars" terraform apply -var-file="../terraform.tfvars" ``` From 5208878f8ef2f6059c46fe38035801a33c7a4f94 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 17 Nov 2022 16:15:41 +0500 Subject: [PATCH 021/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 9d189a503cda..68e33b3a144e 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -152,6 +152,7 @@ terraform apply -var-file="../terraform.tfvars" - `_TAG` *#Tag name for your Playground container images* - `_GKE_NAME` *#Your GKE cluster name for Playground* - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name (configured earlier in terraform.tfvars)* + - `_DOCKER_REPOSITORY_ROOT` *# The name of docker repo root (can be taken from outputs of first trigger result)* 7. Click save. 8. Run the trigger `Playground-to-gke-trigger`. From 61f16079afb9fad2f9a1114f63bd615d42d6c424 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 17 Nov 2022 17:53:09 +0500 Subject: [PATCH 022/269] Update README.md --- .../infrastructure/cloudbuild-manual-setup/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 68e33b3a144e..1dfd9575f8df 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -70,7 +70,7 @@ cd playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/ # Run terraform scripts # Where "bucket=" set your bucket name for TF state -terraform init -backend-config="bucket=playground-state-bucket" +terraform init -backend-config="bucket=playground-state-bucket" terraform plan -var-file="../terraform.tfvars" terraform apply -var-file="../terraform.tfvars" @@ -114,7 +114,7 @@ github_repository_branch = "cloudbuild+manualsetup+playground" # The na cd ../02.builders # Run terraform scripts # Where "bucket=" set your bucket name for TF state -terraform init -backend-config="bucket=playground-state-bucket" +terraform init -backend-config="bucket=playground-state-bucket" terraform plan -var-file="../terraform.tfvars" terraform apply -var-file="../terraform.tfvars" ``` From b983ef7ae2f4b892de37e351f82bdf39af9a624c Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 17 Nov 2022 23:20:51 +0500 Subject: [PATCH 023/269] Updates related to #24144 --- .gitignore | 1 - ...ockerfile-base-image-worker => Dockerfile} | 0 .../cloudbuild/cloudbuild_pg_infra.yaml | 10 +-- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 4 +- .../01.setup/output.tf | 31 -------- .../02.builders/output.tf | 25 ------- .../02.builders/triggers.tf | 2 +- .../02.builders/variables.tf | 4 +- .../cloudbuild-manual-setup/README.md | 70 +++++++------------ 9 files changed, 34 insertions(+), 113 deletions(-) rename playground/infrastructure/cloudbuild/{Dockerfile-base-image-worker => Dockerfile} (100%) delete mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/output.tf delete mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/output.tf diff --git a/.gitignore b/.gitignore index 6b9a09e67570..e67e392ec171 100644 --- a/.gitignore +++ b/.gitignore @@ -137,4 +137,3 @@ website/www/yarn-error.log # Ignore Katas auto-generated files **/*-remote-info.yaml -playground/terraform/infrastructure/cloudbuild-manual-setup/terraform.tfvars diff --git a/playground/infrastructure/cloudbuild/Dockerfile-base-image-worker b/playground/infrastructure/cloudbuild/Dockerfile similarity index 100% rename from playground/infrastructure/cloudbuild/Dockerfile-base-image-worker rename to playground/infrastructure/cloudbuild/Dockerfile diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 1534339613ba..538731f2d604 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -16,17 +16,17 @@ # under the License. steps: - # This stage builds the image from Dockerfile that will be used as Docker environment to run Beam Playground deployment + # This stage builds the image from Dockerfile that will be used as environment to run Beam Playground deployment - name: 'gcr.io/cloud-builders/docker' args: - 'build' - - '--tag=${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/playground-builder:worker' + - '--tag=${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:playground' - '-f' - - 'playground/infrastructure/cloudbuild/Dockerfile-base-image-worker' + - 'playground/infrastructure/cloudbuild/Dockerfile' - '.' # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:worker' + - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:playground' entrypoint: "/bin/bash" args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="${_ENVIRONMENT_NAME}" -Pdns-name="${_DNS_NAME}"'] @@ -44,7 +44,7 @@ logsBucket: 'gs://${_LOGS_BUCKET_NAME}' # Pushing built image to GCP Artifact Registry repository images: - - '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:worker' + - '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:playground' # This option enables writing logs to GCS bucket options: diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 82079a81aef4..c60373854ce4 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -17,7 +17,7 @@ steps: # This stage uses pre-built Docker container to run gradle task for Playground pre-config and deployment to GKE - - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:worker' + - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:playground' entrypoint: "/bin/bash" args: [ @@ -39,7 +39,7 @@ substitutions: # The name of the GCP logs bucket _LOGS_BUCKET_NAME: "pg-cloudbuild-logging-bucket" # The name of docker repository root (from outputs of first trigger result) - _DOCKER_REPOSITORY_ROOT: "" + _DOCKER_REPOSITORY_ROOT: "${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository" # The name of your GCP logs bucket logsBucket: 'gs://${_LOGS_BUCKET_NAME}' diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/output.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/output.tf deleted file mode 100644 index e7405c14ea29..000000000000 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/output.tf +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -output "cloudbuild_service_account_id" { - value = google_service_account.cloudbuild_service_account_id -} - -output "How_to_connect_github_repo_to_cloudbuild" { - value = < Triggers. +2. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. 3. Choose region that was configured earlier in terraform.tfvars. 4. Open Trigger: `Playground-infrastructure-trigger`. 5. Scroll down to `Advanced` - `Substitutions variables`. @@ -132,8 +114,7 @@ terraform apply -var-file="../terraform.tfvars" - `_ENVIRONMENT_NAME` *#Your env name in beam/playground/terraform/environment/* - `_DNS_NAME` *#Your DNS for Playground* - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name (configured earlier in terraform.tfvars)* -8. Click save. -9. Run the trigger `Playground-infrastructure-trigger`. +8. Click `Save` and Run the trigger `Playground-infrastructure-trigger`. 10. Once Playground infrastructure has been deployed, please navigate to [Playground deployment README](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#deploy-playground-infrastructure) and execute step #2: @@ -141,21 +122,18 @@ terraform apply -var-file="../terraform.tfvars" ## 5. Running Second Cloud Build Trigger: "Playground-to-gke-trigger" -1. Navigate to GCP Console. Open Cloud Build page -> Triggers. +1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. 2. Choose region that was configured earlier in terraform.tfvars. 3. Open Trigger: `Playground-to-gke-trigger`. -4. Scroll down to `Advanced` - `Substitutions variables`. -5. Click on `+ ADD VARIABLE` -6. Assign values for next variables: +4. Scroll down to `Advanced` - `Substitutions variables` and Click on `+ ADD VARIABLE`. +5. Assign values for next variables: - `_ENVIRONMENT_NAME` *#Your env name in beam/playground/terraform/environment/* - `_DNS_NAME` *#Your DNS for Playground* - `_TAG` *#Tag name for your Playground container images* - `_GKE_NAME` *#Your GKE cluster name for Playground* - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name (configured earlier in terraform.tfvars)* - `_DOCKER_REPOSITORY_ROOT` *# The name of docker repo root (can be taken from outputs of first trigger result)* -7. Click save. -8. Run the trigger `Playground-to-gke-trigger`. - +6. Click `Save` and Run the trigger `Playground-to-gke-trigger`. ## 6. Validation From c9d965710017344ab6d39fae8990fda677cbc4c4 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 17 Nov 2022 23:22:32 +0500 Subject: [PATCH 024/269] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e67e392ec171..62f5a20d45e7 100644 --- a/.gitignore +++ b/.gitignore @@ -136,4 +136,4 @@ website/www/yarn-error.log **/*.tfvars # Ignore Katas auto-generated files -**/*-remote-info.yaml +**/*-remote-info.yaml \ No newline at end of file From e1dd2ae421c98f8d7c09958c59c6850522d6c37f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 00:35:23 +0500 Subject: [PATCH 025/269] Updates related to PR #24144 --- .../01.setup/bucket.tf | 1 + .../cloudbuild-manual-setup/01.setup/iam.tf | 2 +- .../01.setup/provider.tf | 2 +- .../01.setup/services.tf | 1 + .../01.setup/variables.tf | 2 +- .../02.builders/provider.tf | 2 +- .../02.builders/triggers.tf | 12 +-- .../02.builders/variables.tf | 2 +- .../cloudbuild-manual-setup/README.md | 98 +++++++++---------- 9 files changed, 62 insertions(+), 60 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf index 361f4ac9117b..b8cc96a70b5f 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf @@ -17,6 +17,7 @@ resource "google_storage_bucket" "cloudbuild_logs_bucket" { name = var.cloudbuild_logs_bucket_name + project = var.project_id location = var.region storage_class = "STANDARD" } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf index 74b32013bdc8..3117134ffcc9 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf @@ -40,5 +40,5 @@ resource "google_project_iam_member" "cloud_build_roles" { ]) role = each.key member = "serviceAccount:${google_service_account.cloudbuild_service_account_id.email}" - project = var.project + project = var.project_id } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf index 43ec769c4a14..fccee503e808 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf @@ -16,6 +16,6 @@ # under the License. provider "google" { - project = var.project + project = var.project_id region = var.region } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf index 245009f4a5fa..1821c1512ace 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf @@ -17,6 +17,7 @@ // Provision the required Google Cloud services resource "google_project_service" "required_services" { + project = var.project_id for_each = toset([ "cloudresourcemanager", "cloudbuild", diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf index 3dde46d0fee9..6d86178e9619 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -variable "project" { +variable "project_id" { type = string description = "The ID of the Google Cloud project within which resources are provisioned" } diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf index 43ec769c4a14..fccee503e808 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf @@ -16,6 +16,6 @@ # under the License. provider "google" { - project = var.project + project = var.project_id region = var.region } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf index b7192348eaad..b264a3e0f48c 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf @@ -20,9 +20,9 @@ data "google_service_account" "cloudbuild_sa" { } resource "google_cloudbuild_trigger" "playground_infrastructure" { - name = var.infra_trigger_id + name = var.infra_trigger_name location = var.region - project = var.project + project = var.project_id description = "Builds the base image and then runs cloud build config file to deploy Playground infrastructure" @@ -39,13 +39,13 @@ resource "google_cloudbuild_trigger" "playground_infrastructure" { repo_type = "GITHUB" } - service_account = data.google_service_account.myaccount.id + service_account = data.google_service_account.cloudbuild_sa.id } resource "google_cloudbuild_trigger" "playground_to_gke" { - name = var.gke_trigger_id + name = var.gke_trigger_name location = var.region - project = var.project + project = var.project_id description = "Builds the base image and then runs cloud build config file to deploy Playground to GKE" @@ -62,5 +62,5 @@ resource "google_cloudbuild_trigger" "playground_to_gke" { repo_type = "GITHUB" } - service_account = data.google_service_account.myaccount.id + service_account = data.google_service_account.cloudbuild_sa.id } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf index 0d2390ecb9f0..5013954a140b 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -variable "project" { +variable "project_id" { type = string description = "The ID of the Google Cloud project within which resources are provisioned" } diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 3149b2e589be..a0e675287fc5 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -42,32 +42,39 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi - Required API services - Cloud Build service account - IAM permissions for Cloud Build service account -- GCP Storage Bucket for Cloud Build logs +- GCP Storage Bucket for Cloud Build logs (Note: [name must be globally unique](https://cloud.google.com/storage/docs/buckets#:~:text=Bucket%20names%20reside%20in%20a,responds%20with%20an%20error%20message.)) #### To execute the module: 1. Create a folder `playground/terraform/environment/{environment_name}` (further referred as `environment_name`) to define a new environment for Playground. 2. Create `terraform.tfvars` file in newly created `environment_name` folder. -3. Fill the configuration file: +3. Fill the `terraform.tfvars` configuration file as follows: -* `terraform.tfvars` environment variables: ``` -project = "sandbox-playground-001" # Your Project ID -region = "us-central1" # Your GCP region name where resources will be provisioned -cloudbuild_service_account_id = "terraform-cloudbuild" # Name of SA to be used by Cloud Build to deploy Playground -cloudbuild_logs_bucket_name = "cb_logs_bucket" # The name of GCP bucket for Cloud Build logs -github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' -github_repository_owner = "owner" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. -github_repository_branch = "cloudbuild+manualsetup+playground" # The name of GitHub repo branch +project_id = "project_id" # Your Project ID +region = "us-central1" # Your GCP region name where resources will be provisioned +location = "us-central1-b" # Select the deployment location from available in the specified region +cloudbuild_service_account_id = "cloudbuild_sa_name" # Name of SA to be used by Cloud Build to deploy Playground +cloudbuild_logs_bucket_name = "logs_bucket_name" # Assign globally unique name of GCP bucket for Cloud Build logs +github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' +github_repository_owner = "owner_name" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. +github_repository_branch = "branch_name" # The name of GitHub repo branch + +network_name = "network_name" # GCP VPC Network Name for Playground deployment +gke_name = "playground-backend" # Playground GKE Cluster name +state_bucket = "bucket_name" # GCS bucket name for Beam Playground temp files +bucket_examples_name = "bucket_name-example" # GCS bucket name for Playground examples storage ``` -2. Run commands: +**Note:** Some regions can be prohibited for Cloud Build. Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for supported locations. + +4. Run commands: ```console -# Creates new authentication configuration to GCP Project with user account created in Requirements +# Create new authentication configuration to GCP Project with created user account gcloud init -# Acquires new user account credentials to use for Application Default Credentials +# Command imports new user account credentials into Application Default Credentials gcloud auth application-default login # Navigate to 01.setup folder @@ -87,54 +94,47 @@ Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/auto The `playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders` provisions: - Cloud Build triggers to build and deploy Beam Playground. -#### Execute the module - -1. Run: +#### To execute the module ``` # Navigate to playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders folder cd ../02.builders + # Run terraform scripts -# Where "bucket=" set your bucket name for TF state -terraform init -backend-config="bucket=playground-state-bucket" +terraform init -backend-config="bucket=state_bucket" terraform plan -var-file="../../environment/{environment_name}/terraform.tfvars" terraform apply -var-file="../../environment/{environment_name}/terraform.tfvars" ``` -## 4. Running First Cloud Build Trigger: "Playground-infrastructure-trigger" - -1. Perform steps described in [Prepare deployment configuration](https://github.com/apache/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#prepare-deployment-configuration) (Add required variables to existing .tfvars file). - -2. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. -3. Choose region that was configured earlier in terraform.tfvars. -4. Open Trigger: `Playground-infrastructure-trigger`. -5. Scroll down to `Advanced` - `Substitutions variables`. -6. Click on `+ ADD VARIABLE` -7. Assign values for next variables: - - `_ENVIRONMENT_NAME` *#Your env name in beam/playground/terraform/environment/* - - `_DNS_NAME` *#Your DNS for Playground* - - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name (configured earlier in terraform.tfvars)* -8. Click `Save` and Run the trigger `Playground-infrastructure-trigger`. - -10. Once Playground infrastructure has been deployed, please navigate to -[Playground deployment README](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#deploy-playground-infrastructure) and execute step #2: -`Add following DNS A records for the discovered static IP address` - -## 5. Running Second Cloud Build Trigger: "Playground-to-gke-trigger" +## 4. Running first Cloud Build trigger to deploy Playground infrastructure -1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. -2. Choose region that was configured earlier in terraform.tfvars. -3. Open Trigger: `Playground-to-gke-trigger`. +1. Perform steps described in [Prepare deployment configuration](https://github.com/apache/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#prepare-deployment-configuration) +2. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (Example: us-central1). +3. Open Trigger: `Playground-infrastructure-trigger`. 4. Scroll down to `Advanced` - `Substitutions variables` and Click on `+ ADD VARIABLE`. 5. Assign values for next variables: - - `_ENVIRONMENT_NAME` *#Your env name in beam/playground/terraform/environment/* - - `_DNS_NAME` *#Your DNS for Playground* - - `_TAG` *#Tag name for your Playground container images* - - `_GKE_NAME` *#Your GKE cluster name for Playground* - - `_LOGS_BUCKET_NAME` *#Your GCP logs bucket name (configured earlier in terraform.tfvars)* - - `_DOCKER_REPOSITORY_ROOT` *# The name of docker repo root (can be taken from outputs of first trigger result)* -6. Click `Save` and Run the trigger `Playground-to-gke-trigger`. + - `_ENVIRONMENT_NAME`           # *Created "environment_name"* + - `_DNS_NAME`                         *# Your DNS record name* + - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name (configured in terraform.tfvars)* +6. Click `Save` and Run the trigger `Playground-infrastructure-trigger`. + +7. Once Playground infrastructure has been deployed, please navigate to + [Playground deployment README](https://github.com/apache/beam/tree/master/playground/terraform#deploy-playground-infrastructure) and execute step #2: + `Add following DNS A records for the discovered static IP address` + +## 5. Running second Cloud Build trigger to deploy Playground to GKE + +1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (Example: us-central1). +2. Open Trigger: `Playground-to-gke-trigger`. +3. Scroll down to `Advanced` - `Substitutions variables` and Click on `+ ADD VARIABLE`. +4. Assign values for next variables: + - `_ENVIRONMENT_NAME`           # *Created "environment_name"* + - `_DNS_NAME`                         *# Your DNS record name* + - `_TAG`                                 *# Tag name for your Playground container images* + - `_GKE_NAME`                         *# Your GKE cluster name (configured in terraform.tfvars)* + - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name (configured in terraform.tfvars)* +5. Click `Save` and Run the trigger `Playground-to-gke-trigger`. ## 6. Validation -Once Playground has been deployed to GKE, please navigate to [Validation](https://github.com/akvelon/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file +Once Playground has been deployed to GKE, please navigate to [Validation](https://github.com/apache/beam/tree/master/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file From a4c36da028874a9ea1b923d74f5f6d4f98a4f9ae Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 00:45:15 +0500 Subject: [PATCH 026/269] Update README.md --- .../cloudbuild-manual-setup/README.md | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index a0e675287fc5..8fc0fd3c91b2 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -17,13 +17,13 @@ # Beam Playground Cloud Build Setup -This directory organizes Infrastructure-as-Code to provision dependent resources and setup Cloud Build for Beam Playground. +This directory organizes Infrastructure-as-Code to provision dependent resources and set up Cloud Build for Beam Playground. ## Requirements: - [GCP project](https://cloud.google.com/resource-manager/docs/creating-managing-projects) - [GCP User account](https://cloud.google.com/appengine/docs/standard/access-control?tab=python) _(Note: You will find the instruction "How to create User account" for your new project)_
- Ensure that the account has at least following privileges: + Ensure that the account has at least the following privileges: - Service Account Admin - Storage Admin - Service Usage Admin @@ -46,8 +46,8 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi #### To execute the module: -1. Create a folder `playground/terraform/environment/{environment_name}` (further referred as `environment_name`) to define a new environment for Playground. -2. Create `terraform.tfvars` file in newly created `environment_name` folder. +1. Create a folder `playground/terraform/environment/{environment_name}` (further referred to as `environment_name`) to define a new environment for Playground. +2. Create `terraform.tfvars` file in the newly created `environment_name` folder. 3. Fill the `terraform.tfvars` configuration file as follows: ``` @@ -55,10 +55,10 @@ project_id = "project_id" # Your Project ID region = "us-central1" # Your GCP region name where resources will be provisioned location = "us-central1-b" # Select the deployment location from available in the specified region cloudbuild_service_account_id = "cloudbuild_sa_name" # Name of SA to be used by Cloud Build to deploy Playground -cloudbuild_logs_bucket_name = "logs_bucket_name" # Assign globally unique name of GCP bucket for Cloud Build logs +cloudbuild_logs_bucket_name = "logs_bucket_name" # Assign a globally unique name of GCP bucket for Cloud Build logs github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' github_repository_owner = "owner_name" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. -github_repository_branch = "branch_name" # The name of GitHub repo branch +github_repository_branch = "branch_name" # The name of the GitHub repo branch network_name = "network_name" # GCP VPC Network Name for Playground deployment gke_name = "playground-backend" # Playground GKE Cluster name @@ -66,12 +66,14 @@ state_bucket = "bucket_name" # GCS bucket name fo bucket_examples_name = "bucket_name-example" # GCS bucket name for Playground examples storage ``` +5. Push the `environment_name` folder to your repository. + **Note:** Some regions can be prohibited for Cloud Build. Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for supported locations. 4. Run commands: ```console -# Create new authentication configuration to GCP Project with created user account +# Create a new authentication configuration for GCP Project with the created user account gcloud init # Command imports new user account credentials into Application Default Credentials @@ -87,7 +89,7 @@ terraform apply -var-file="../../environment/{environment_name}/terraform.tfvars ``` ## 2. Connect GitHub repository and Cloud Build -Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github) to connect GitHub repository and Cloud Build. +Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github) to connect the GitHub repository and Cloud Build. ## 3. Setup the Google Cloud Build triggers @@ -106,13 +108,13 @@ terraform plan -var-file="../../environment/{environment_name}/terraform.tfvars" terraform apply -var-file="../../environment/{environment_name}/terraform.tfvars" ``` -## 4. Running first Cloud Build trigger to deploy Playground infrastructure +## 4. Running the first Cloud Build trigger to deploy Playground infrastructure -1. Perform steps described in [Prepare deployment configuration](https://github.com/apache/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#prepare-deployment-configuration) +1. Perform the steps described in [Prepare deployment configuration](https://github.com/apache/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#prepare-deployment-configuration) 2. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (Example: us-central1). 3. Open Trigger: `Playground-infrastructure-trigger`. 4. Scroll down to `Advanced` - `Substitutions variables` and Click on `+ ADD VARIABLE`. -5. Assign values for next variables: +5. Assign values for the next variables: - `_ENVIRONMENT_NAME`           # *Created "environment_name"* - `_DNS_NAME`                         *# Your DNS record name* - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name (configured in terraform.tfvars)* @@ -122,12 +124,12 @@ terraform apply -var-file="../../environment/{environment_name}/terraform.tfvars [Playground deployment README](https://github.com/apache/beam/tree/master/playground/terraform#deploy-playground-infrastructure) and execute step #2: `Add following DNS A records for the discovered static IP address` -## 5. Running second Cloud Build trigger to deploy Playground to GKE +## 5. Running the second Cloud Build trigger to deploy Playground to GKE 1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (Example: us-central1). 2. Open Trigger: `Playground-to-gke-trigger`. 3. Scroll down to `Advanced` - `Substitutions variables` and Click on `+ ADD VARIABLE`. -4. Assign values for next variables: +4. Assign values for the next variables: - `_ENVIRONMENT_NAME`           # *Created "environment_name"* - `_DNS_NAME`                         *# Your DNS record name* - `_TAG`                                 *# Tag name for your Playground container images* From 8c62e16ca4483144f5885015932432dd30ed7146 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 16:33:17 +0500 Subject: [PATCH 027/269] Updates for apache #24144 --- .../01.setup/bucket.tf | 23 ------- .../01.setup/variables.tf | 6 +- .../cloudbuild-manual-setup/README.md | 62 +++++++++++-------- 3 files changed, 37 insertions(+), 54 deletions(-) delete mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf deleted file mode 100644 index b8cc96a70b5f..000000000000 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/bucket.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -resource "google_storage_bucket" "cloudbuild_logs_bucket" { - name = var.cloudbuild_logs_bucket_name - project = var.project_id - location = var.region - storage_class = "STANDARD" -} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf index 6d86178e9619..7bc84e31f55e 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf @@ -28,9 +28,5 @@ variable "region" { variable "cloudbuild_service_account_id" { type = string description = "The ID of the cloud build service account responsible for provisioning Google Cloud resources" + default = "playground-cloudbuild-sa" } - -variable "cloudbuild_logs_bucket_name" { - type = string - description = "Globally unique ID of your logs bucket for Cloud Build" -} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 8fc0fd3c91b2..649f71c6369a 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -32,6 +32,7 @@ This directory organizes Infrastructure-as-Code to provision dependent resources - Service Account User - [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) - An existing GCP Bucket to save Terraform state - `state_bucket` +- An existing GCP Bucket to save Cloud Build logs - `logs_bucket` - DNS name for your Playground deployment instance - [Terraform](https://www.terraform.io/) - [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally @@ -40,9 +41,8 @@ This directory organizes Infrastructure-as-Code to provision dependent resources The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provisions dependencies required to setup Cloud Build for Playground: - Required API services -- Cloud Build service account -- IAM permissions for Cloud Build service account -- GCP Storage Bucket for Cloud Build logs (Note: [name must be globally unique](https://cloud.google.com/storage/docs/buckets#:~:text=Bucket%20names%20reside%20in%20a,responds%20with%20an%20error%20message.)) +- Cloud Build service account - `playground-cloudbuild-sa` +- IAM permissions for Cloud Build service account - `playground-cloudbuild-sa` #### To execute the module: @@ -50,27 +50,33 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi 2. Create `terraform.tfvars` file in the newly created `environment_name` folder. 3. Fill the `terraform.tfvars` configuration file as follows: -``` + +```console project_id = "project_id" # Your Project ID region = "us-central1" # Your GCP region name where resources will be provisioned location = "us-central1-b" # Select the deployment location from available in the specified region -cloudbuild_service_account_id = "cloudbuild_sa_name" # Name of SA to be used by Cloud Build to deploy Playground -cloudbuild_logs_bucket_name = "logs_bucket_name" # Assign a globally unique name of GCP bucket for Cloud Build logs -github_repository_name = "beam" # The name of the GitHub repo. Example: In https://github.com/example/foo is 'foo' -github_repository_owner = "owner_name" # Owner of the GitHub repo. Example: In https://github.com/example/foo is 'example'. -github_repository_branch = "branch_name" # The name of the GitHub repo branch +github_repository_name = "beam" # The name of the GitHub repo to be connected with Cloud Build. Example: In https://github.com/example/foo is 'foo' +github_repository_owner = "repo_owner_name" # Owner of the GitHub repo to be connected with Cloud Build. Example: In https://github.com/example/foo is 'example'. +github_repository_branch = "branch_name" # The name of the GitHub repo branch to be connected with Cloud Build network_name = "network_name" # GCP VPC Network Name for Playground deployment gke_name = "playground-backend" # Playground GKE Cluster name state_bucket = "bucket_name" # GCS bucket name for Beam Playground temp files -bucket_examples_name = "bucket_name-example" # GCS bucket name for Playground examples storage ``` -5. Push the `environment_name` folder to your repository. +4. Create and fill `state.tfbackend` configuration file as follows: + + +``` +bucket = "bucket_name" # The name of bucket - will be used for terraform tfstate file +``` + +5. Push the `environment_name` folder to your GitHub repository `github_repository_name` and branch `github_repository_branch`. **Note:** Some regions can be prohibited for Cloud Build. Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for supported locations. -4. Run commands: +6. Run commands: + ```console # Create a new authentication configuration for GCP Project with the created user account @@ -83,13 +89,17 @@ gcloud auth application-default login cd playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/ # Run terraform scripts -terraform init -backend-config="bucket=state_bucket" -terraform plan -var-file="../../environment/{environment_name}/terraform.tfvars" -terraform apply -var-file="../../environment/{environment_name}/terraform.tfvars" +terraform init -backend-config="../../../environment/{environment_name}/state.tfbackend" +terraform plan -var-file="../../../environment/{environment_name}/terraform.tfvars" +terraform apply -var-file="../../../environment/{environment_name}/terraform.tfvars" ``` ## 2. Connect GitHub repository and Cloud Build -Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github) to connect the GitHub repository and Cloud Build. + +**Note:** Ensure correct `region` is set in [Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page before proceeding further. + +Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github) +to connect the GitHub repository `https://github.com/{github_repository_owner}/{github_repository_name}` and `{github_branch_name}` with Cloud Build. ## 3. Setup the Google Cloud Build triggers @@ -98,29 +108,29 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders` pr #### To execute the module + ``` # Navigate to playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders folder cd ../02.builders # Run terraform scripts -terraform init -backend-config="bucket=state_bucket" -terraform plan -var-file="../../environment/{environment_name}/terraform.tfvars" -terraform apply -var-file="../../environment/{environment_name}/terraform.tfvars" +terraform init -backend-config="../../../environment/{environment_name}/state.tfbackend" +terraform plan -var-file="../../../environment/{environment_name}/terraform.tfvars" +terraform apply -var-file="../../../environment/{environment_name}/terraform.tfvars" ``` ## 4. Running the first Cloud Build trigger to deploy Playground infrastructure -1. Perform the steps described in [Prepare deployment configuration](https://github.com/apache/beam/tree/Infra%2Bplayground-in-gke/playground/terraform#prepare-deployment-configuration) -2. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (Example: us-central1). -3. Open Trigger: `Playground-infrastructure-trigger`. -4. Scroll down to `Advanced` - `Substitutions variables` and Click on `+ ADD VARIABLE`. -5. Assign values for the next variables: +1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (Example: us-central1). +2. Open Trigger: `Playground-infrastructure-trigger`. +3. Scroll down to `Advanced` - `Substitutions variables` and Click on `+ ADD VARIABLE`. +4. Assign values for the next variables: - `_ENVIRONMENT_NAME`           # *Created "environment_name"* - `_DNS_NAME`                         *# Your DNS record name* - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name (configured in terraform.tfvars)* -6. Click `Save` and Run the trigger `Playground-infrastructure-trigger`. +5. Click `Save` and Run the trigger `Playground-infrastructure-trigger`. -7. Once Playground infrastructure has been deployed, please navigate to +6. Once Playground infrastructure has been deployed, please navigate to [Playground deployment README](https://github.com/apache/beam/tree/master/playground/terraform#deploy-playground-infrastructure) and execute step #2: `Add following DNS A records for the discovered static IP address` From aab42a966219e4beb5091fc5fa06329da2111178 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 17:25:28 +0500 Subject: [PATCH 028/269] Update README.md --- .../infrastructure/cloudbuild-manual-setup/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 649f71c6369a..d57917d158ea 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -127,7 +127,7 @@ terraform apply -var-file="../../../environment/{environment_name}/terraform.tfv 4. Assign values for the next variables: - `_ENVIRONMENT_NAME`           # *Created "environment_name"* - `_DNS_NAME`                         *# Your DNS record name* - - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name (configured in terraform.tfvars)* + - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name `logs_bucket`* 5. Click `Save` and Run the trigger `Playground-infrastructure-trigger`. 6. Once Playground infrastructure has been deployed, please navigate to @@ -144,7 +144,7 @@ terraform apply -var-file="../../../environment/{environment_name}/terraform.tfv - `_DNS_NAME`                         *# Your DNS record name* - `_TAG`                                 *# Tag name for your Playground container images* - `_GKE_NAME`                         *# Your GKE cluster name (configured in terraform.tfvars)* - - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name (configured in terraform.tfvars)* + - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name `logs_bucket`* 5. Click `Save` and Run the trigger `Playground-to-gke-trigger`. ## 6. Validation From 978849186b8d34bda9a1a629242c4918d118c723 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 18:55:33 +0500 Subject: [PATCH 029/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index d57917d158ea..ffc4adafc4bd 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -55,6 +55,7 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi project_id = "project_id" # Your Project ID region = "us-central1" # Your GCP region name where resources will be provisioned location = "us-central1-b" # Select the deployment location from available in the specified region +cloudbuild_service_account_id = "playground-cloudbuild-sa" # The name of Cloud Build service account github_repository_name = "beam" # The name of the GitHub repo to be connected with Cloud Build. Example: In https://github.com/example/foo is 'foo' github_repository_owner = "repo_owner_name" # Owner of the GitHub repo to be connected with Cloud Build. Example: In https://github.com/example/foo is 'example'. github_repository_branch = "branch_name" # The name of the GitHub repo branch to be connected with Cloud Build From cb77aca1e7760dafd5efdbba6e6f017a9fca45c0 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 19:15:13 +0500 Subject: [PATCH 030/269] Update variables.tf --- .../cloudbuild-manual-setup/02.builders/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf index 5013954a140b..54e1dc676e9f 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf @@ -55,4 +55,5 @@ variable "gke_trigger_name" { variable "cloudbuild_service_account_id" { type = string description = "The ID of the cloud build service account responsible for provisioning Google Cloud resources" + default = "playground-cloudbuild-sa" } \ No newline at end of file From 0aa6f1ffd81854e950b35ae5b36337e0eb693372 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 20:18:15 +0500 Subject: [PATCH 031/269] Update state.tfbackend --- playground/terraform/environment/beta/state.tfbackend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/environment/beta/state.tfbackend b/playground/terraform/environment/beta/state.tfbackend index b2c574de8e4b..052fd34f9a97 100644 --- a/playground/terraform/environment/beta/state.tfbackend +++ b/playground/terraform/environment/beta/state.tfbackend @@ -17,4 +17,4 @@ # under the License. # -bucket = "pg-second" +bucket = "my-tfstate-bucket-1" From 01e69dd2fb6934d99e0c53b549a850a949e51e92 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 20:25:06 +0500 Subject: [PATCH 032/269] Revert "Update state.tfbackend" This reverts commit 0aa6f1ffd81854e950b35ae5b36337e0eb693372. --- playground/terraform/environment/beta/state.tfbackend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/environment/beta/state.tfbackend b/playground/terraform/environment/beta/state.tfbackend index 052fd34f9a97..b2c574de8e4b 100644 --- a/playground/terraform/environment/beta/state.tfbackend +++ b/playground/terraform/environment/beta/state.tfbackend @@ -17,4 +17,4 @@ # under the License. # -bucket = "my-tfstate-bucket-1" +bucket = "pg-second" From 3a4cbb61f32152786d9dcd59ffd655fbab6215a3 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 20:32:20 +0500 Subject: [PATCH 033/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index ffc4adafc4bd..7369a3a91458 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -37,7 +37,7 @@ This directory organizes Infrastructure-as-Code to provision dependent resources - [Terraform](https://www.terraform.io/) - [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally -## 1. Setup the Google Cloud Build for your GCP project +## 1. Setup the Google Cloud Build for your GCP project The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provisions dependencies required to setup Cloud Build for Playground: - Required API services @@ -73,6 +73,7 @@ bucket = "bucket_name" # The name of bucket - will be used for terraform tfs ``` 5. Push the `environment_name` folder to your GitHub repository `github_repository_name` and branch `github_repository_branch`. +(Cloud Build needs file be present in GitHub repository it is connected to) **Note:** Some regions can be prohibited for Cloud Build. Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for supported locations. From af4de6bbdc5768231b2aa0a6f9f71dd93a32f337 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 20:35:09 +0500 Subject: [PATCH 034/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 7369a3a91458..b11cbf7c27f3 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -65,7 +65,7 @@ gke_name = "playground-backend" # Playground GKE Clu state_bucket = "bucket_name" # GCS bucket name for Beam Playground temp files ``` -4. Create and fill `state.tfbackend` configuration file as follows: +4. Create and fill `state.tfbackend` configuration file in same `environment_name` folder as follows: ``` From a2fc91f1642bd42cf4cd357a9182db15d3ae479c Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 21:44:53 +0500 Subject: [PATCH 035/269] Update README.md --- .../infrastructure/cloudbuild-manual-setup/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index b11cbf7c27f3..a2be2921547e 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -75,7 +75,7 @@ bucket = "bucket_name" # The name of bucket - will be used for terraform tfs 5. Push the `environment_name` folder to your GitHub repository `github_repository_name` and branch `github_repository_branch`. (Cloud Build needs file be present in GitHub repository it is connected to) -**Note:** Some regions can be prohibited for Cloud Build. Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for supported locations. +**Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. 6. Run commands: @@ -121,7 +121,7 @@ terraform plan -var-file="../../../environment/{environment_name}/terraform.tfva terraform apply -var-file="../../../environment/{environment_name}/terraform.tfvars" ``` -## 4. Running the first Cloud Build trigger to deploy Playground infrastructure +## 4. Run Cloud Build `Playground-infrastructure-trigger` to deploy Playground infrastructure 1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (Example: us-central1). 2. Open Trigger: `Playground-infrastructure-trigger`. @@ -136,7 +136,7 @@ terraform apply -var-file="../../../environment/{environment_name}/terraform.tfv [Playground deployment README](https://github.com/apache/beam/tree/master/playground/terraform#deploy-playground-infrastructure) and execute step #2: `Add following DNS A records for the discovered static IP address` -## 5. Running the second Cloud Build trigger to deploy Playground to GKE +## 5. Run Cloud Build `Playground-to-gke-trigger` to deploy Playground to GKE 1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (Example: us-central1). 2. Open Trigger: `Playground-to-gke-trigger`. @@ -149,6 +149,6 @@ terraform apply -var-file="../../../environment/{environment_name}/terraform.tfv - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name `logs_bucket`* 5. Click `Save` and Run the trigger `Playground-to-gke-trigger`. -## 6. Validation +## 6. Validate Playground deployment Once Playground has been deployed to GKE, please navigate to [Validation](https://github.com/apache/beam/tree/master/playground/terraform#validate-deployed-playground) to perform Playground deployment steps. \ No newline at end of file From ff0b581b2e4277b83d78d73b3854627975e302c3 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 18 Nov 2022 22:19:30 +0500 Subject: [PATCH 036/269] Update variables and logging type #24144 --- .../cloudbuild/cloudbuild_pg_infra.yaml | 17 ++++++--------- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 21 +++++++------------ 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 538731f2d604..1f79a3e4f1b3 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -20,15 +20,15 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: - 'build' - - '--tag=${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:playground' + - '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' - '-f' - 'playground/infrastructure/cloudbuild/Dockerfile' - '.' # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:playground' + - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' entrypoint: "/bin/bash" - args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="${_ENVIRONMENT_NAME}" -Pdns-name="${_DNS_NAME}"'] + args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] # Substitutions can be changed in GCP Cloud Build Triggers UI substitutions: @@ -36,18 +36,13 @@ substitutions: _ENVIRONMENT_NAME: "test" # The name of your DNS for Playground _DNS_NAME: "dev-playground.online" - # The name of the GCP logs bucket - _LOGS_BUCKET_NAME: "pg-cloudbuild-logging-bucket" - -# The name of your GCP logs bucket -logsBucket: 'gs://${_LOGS_BUCKET_NAME}' # Pushing built image to GCP Artifact Registry repository images: - - '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:playground' + - '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' -# This option enables writing logs to GCS bucket +# This option enables writing logs to Cloud Logging options: - logging: GCS_ONLY + logging: CLOUD_LOGGING_ONLY timeout: 1800s \ No newline at end of file diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index c60373854ce4..d874709411ec 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -17,13 +17,13 @@ steps: # This stage uses pre-built Docker container to run gradle task for Playground pre-config and deployment to GKE - - name: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository/builder:playground' + - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' entrypoint: "/bin/bash" args: [ - '-c', './gradlew playground:terraform:prepareConfig -Pdns-name="${_DNS_NAME}" && - ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="${_DOCKER_REPOSITORY_ROOT}" - -Pproject_environment="${_ENVIRONMENT_NAME}" -Pdocker-tag="${_TAG}" -Pdns-name="${_DNS_NAME}"' + '-c', './gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" && + ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" + -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Pdns-name="$_DNS_NAME"' ] # Substitutions can be changed in GCP Cloud Build Triggers UI @@ -33,19 +33,14 @@ substitutions: # The name of your DNS for Playground _DNS_NAME: "dev-playground.online" # The name of your Docker tag - _TAG: "tag-playground" + _TAG: "playground" # The name of your GKE cluster (from terraform.tfvars) _GKE_NAME: "playground-cluster" - # The name of the GCP logs bucket - _LOGS_BUCKET_NAME: "pg-cloudbuild-logging-bucket" # The name of docker repository root (from outputs of first trigger result) - _DOCKER_REPOSITORY_ROOT: "${LOCATION}-docker.pkg.dev/${PROJECT_ID}/playground-repository" + _DOCKER_REPOSITORY_ROOT: "$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository" -# The name of your GCP logs bucket -logsBucket: 'gs://${_LOGS_BUCKET_NAME}' - -# This option enables writing logs to GCS bucket +# This option enables writing logs to Cloud Logging options: - logging: GCS_ONLY + logging: CLOUD_LOGGING_ONLY timeout: 3600s \ No newline at end of file From 25c25ba671c3eb82c66304a8235e9029c6f6735b Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 21 Nov 2022 21:24:04 +0500 Subject: [PATCH 037/269] Updates related to tfvars file and Readme --- .../cloudbuild/cloudbuild_pg_infra.yaml | 22 ++++--- .../01.setup/provider.tf | 1 - .../02.builders/provider.tf | 1 - .../02.builders/triggers.tf | 32 ++++++++--- .../02.builders/variables.tf | 31 +++++----- .../cloudbuild-manual-setup/README.md | 57 ++++++------------- 6 files changed, 69 insertions(+), 75 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 1f79a3e4f1b3..498ceb98abbb 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -28,14 +28,22 @@ steps: # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' entrypoint: "/bin/bash" - args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] + args: + - '-c' + - | + mkdir playground/terraform/environment/$_ENVIRONMENT_NAME + printf '%s\n' 'project_id = $PROJECT_ID' \ + 'network_name = $_NETWORK_NAME' \ + 'gke_name = $_GKE_NAME' \ + 'region = $LOCATION' \ + 'location = $LOCATION-a' \ + 'state_bucket = $_STATE_BUCKET' \ + > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars -# Substitutions can be changed in GCP Cloud Build Triggers UI -substitutions: - # The name of your environment in beam/playground/terraform/environment/ - _ENVIRONMENT_NAME: "test" - # The name of your DNS for Playground - _DNS_NAME: "dev-playground.online" +# This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment + - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' + entrypoint: "/bin/bash" + args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] # Pushing built image to GCP Artifact Registry repository images: diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf index fccee503e808..c23ddd6f9bf6 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf @@ -17,5 +17,4 @@ provider "google" { project = var.project_id - region = var.region } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf index fccee503e808..c23ddd6f9bf6 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf @@ -17,5 +17,4 @@ provider "google" { project = var.project_id - region = var.region } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf index b264a3e0f48c..248e26ad4e03 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf @@ -27,18 +27,24 @@ resource "google_cloudbuild_trigger" "playground_infrastructure" { description = "Builds the base image and then runs cloud build config file to deploy Playground infrastructure" source_to_build { - uri = "https://github.com/${var.github_repository_owner}/${var.github_repository_name}" - ref = "refs/heads/${var.github_repository_branch}" + uri = "https://github.com/apache/beam" + ref = "refs/heads/master" repo_type = "GITHUB" } git_file_source { path = "playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml" - uri = "https://github.com/${var.github_repository_owner}/${var.github_repository_name}" - revision = "refs/heads/${var.github_repository_branch}" repo_type = "GITHUB" } + substitutions = { + _ENVIRONMENT_NAME: var.playground_environment_name + _DNS_NAME: var.playground_dns_name + _NETWORK_NAME: var.playground_network_name + _GKE_NAME: var.playground_gke_name + _STATE_BUCKET: var.state_bucket + } + service_account = data.google_service_account.cloudbuild_sa.id } @@ -50,17 +56,25 @@ resource "google_cloudbuild_trigger" "playground_to_gke" { description = "Builds the base image and then runs cloud build config file to deploy Playground to GKE" source_to_build { - uri = "https://github.com/${var.github_repository_owner}/${var.github_repository_name}" - ref = "refs/heads/${var.github_repository_branch}" + uri = "https://github.com/apache/beam" + ref = "refs/heads/master" repo_type = "GITHUB" } git_file_source { - path = "playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml" - uri = "https://github.com/${var.github_repository_owner}/${var.github_repository_name}" - revision = "refs/heads/${var.github_repository_branch}" + path = "playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml" repo_type = "GITHUB" } + substitutions = { + _ENVIRONMENT_NAME: var.playground_environment_name + _DNS_NAME: var.playground_dns_name + _NETWORK_NAME: var.playground_network_name + _GKE_NAME: var.playground_gke_name + _STATE_BUCKET: var.state_bucket + _TAG: var.image_tag + _DOCKER_REPOSITORY_ROOT: var.docker_repository_root + } + service_account = data.google_service_account.cloudbuild_sa.id } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf index 54e1dc676e9f..0795a1e98f96 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf @@ -25,21 +25,6 @@ variable "region" { description = "The Google Cloud Platform (GCP) region in which to provision resources" } -variable "github_repository_name" { - type = string - description = "The name of the GitHub repository. For example the repository name for https://github.com/example/foo is 'foo'." -} - -variable "github_repository_owner" { - type = string - description = "Owner of the GitHub repository. For example the owner for https://github.com/example/foo is 'example'." -} - -variable "github_repository_branch" { - type = string - description = "The GitHub repository branch name to match cloud build trigger" -} - variable "infra_trigger_name" { type = string description = "The name of the trigger that will deploy Playground infrastructure" @@ -56,4 +41,18 @@ variable "cloudbuild_service_account_id" { type = string description = "The ID of the cloud build service account responsible for provisioning Google Cloud resources" default = "playground-cloudbuild-sa" -} \ No newline at end of file +} + +variable "playground_environment_name" {} + +variable "playground_dns_name" {} + +variable "playground_network_name" {} + +variable "playground_gke_name" {} + +variable "state_bucket" {} + +variable "image_tag" {} + +variable "docker_repository_root" {} \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index a2be2921547e..2a822b6721c2 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -32,52 +32,30 @@ This directory organizes Infrastructure-as-Code to provision dependent resources - Service Account User - [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) - An existing GCP Bucket to save Terraform state - `state_bucket` -- An existing GCP Bucket to save Cloud Build logs - `logs_bucket` - DNS name for your Playground deployment instance - [Terraform](https://www.terraform.io/) - [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally -## 1. Setup the Google Cloud Build for your GCP project +## 1. Set up the Google Cloud Build for your GCP project -The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provisions dependencies required to setup Cloud Build for Playground: +The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provisions dependencies required to set up Cloud Build for Playground: - Required API services - Cloud Build service account - `playground-cloudbuild-sa` - IAM permissions for Cloud Build service account - `playground-cloudbuild-sa` #### To execute the module: -1. Create a folder `playground/terraform/environment/{environment_name}` (further referred to as `environment_name`) to define a new environment for Playground. -2. Create `terraform.tfvars` file in the newly created `environment_name` folder. -3. Fill the `terraform.tfvars` configuration file as follows: - +1. Set following environment variables: + - STATE_BUCKET - GCP Storage bucket name to save Terraform state + - GOOGLE_PROJECT - GCP Project ID + - GOOGLE_REGION - GCP region to save resources to ```console -project_id = "project_id" # Your Project ID -region = "us-central1" # Your GCP region name where resources will be provisioned -location = "us-central1-b" # Select the deployment location from available in the specified region -cloudbuild_service_account_id = "playground-cloudbuild-sa" # The name of Cloud Build service account -github_repository_name = "beam" # The name of the GitHub repo to be connected with Cloud Build. Example: In https://github.com/example/foo is 'foo' -github_repository_owner = "repo_owner_name" # Owner of the GitHub repo to be connected with Cloud Build. Example: In https://github.com/example/foo is 'example'. -github_repository_branch = "branch_name" # The name of the GitHub repo branch to be connected with Cloud Build - -network_name = "network_name" # GCP VPC Network Name for Playground deployment -gke_name = "playground-backend" # Playground GKE Cluster name -state_bucket = "bucket_name" # GCS bucket name for Beam Playground temp files -``` - -4. Create and fill `state.tfbackend` configuration file in same `environment_name` folder as follows: - - -``` -bucket = "bucket_name" # The name of bucket - will be used for terraform tfstate file + export STATE_BUCKET="state_bucket" GOOGLE_PROJECT="project_id" GOOGLE_REGION="region" ``` - -5. Push the `environment_name` folder to your GitHub repository `github_repository_name` and branch `github_repository_branch`. -(Cloud Build needs file be present in GitHub repository it is connected to) - **Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. -6. Run commands: +2. Run commands: ```console @@ -91,22 +69,20 @@ gcloud auth application-default login cd playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/ # Run terraform scripts -terraform init -backend-config="../../../environment/{environment_name}/state.tfbackend" -terraform plan -var-file="../../../environment/{environment_name}/terraform.tfvars" -terraform apply -var-file="../../../environment/{environment_name}/terraform.tfvars" +terraform init -backend-config="bucket=$STATE_BUCKET" +terraform apply -var "project_id=$GOOGLE_PROJECT" ``` -## 2. Connect GitHub repository and Cloud Build +## 2. Connect Apache Beam GitHub repository and GCP Cloud Build **Note:** Ensure correct `region` is set in [Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page before proceeding further. -Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github) -to connect the GitHub repository `https://github.com/{github_repository_owner}/{github_repository_name}` and `{github_branch_name}` with Cloud Build. +Follow [Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github) to connect Apache Beam GitHub repository and GCP Cloud Build. -## 3. Setup the Google Cloud Build triggers +## 3. Set up the Google Cloud Build triggers The `playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders` provisions: -- Cloud Build triggers to build and deploy Beam Playground. +- Cloud Build triggers to build and deploy Beam Playground #### To execute the module @@ -116,9 +92,8 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders` pr cd ../02.builders # Run terraform scripts -terraform init -backend-config="../../../environment/{environment_name}/state.tfbackend" -terraform plan -var-file="../../../environment/{environment_name}/terraform.tfvars" -terraform apply -var-file="../../../environment/{environment_name}/terraform.tfvars" +terraform init -backend-config="bucket=$STATE_BUCKET" +terraform apply -var "project_id=$GOOGLE_PROJECT" -var "region=$GOOGLE_REGION" ``` ## 4. Run Cloud Build `Playground-infrastructure-trigger` to deploy Playground infrastructure From 1989f8cb1270bd58da26005fb18e284a3a84fed7 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 22 Nov 2022 01:31:35 +0500 Subject: [PATCH 038/269] Updates related to apache beam PR #24144 --- .../cloudbuild-manual-setup/01.setup/iam.tf | 3 +- .../01.setup/variables.tf | 5 ---- .../02.builders/variables.tf | 28 ++++++++++++++----- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf index 3117134ffcc9..f961f32af719 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf @@ -35,8 +35,7 @@ resource "google_project_iam_member" "cloud_build_roles" { "roles/iam.securityAdmin", "roles/iam.serviceAccountUser", "roles/datastore.indexAdmin", - "roles/storage.admin", - "roles/logging.logWriter" + "roles/storage.admin" ]) role = each.key member = "serviceAccount:${google_service_account.cloudbuild_service_account_id.email}" diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf index 7bc84e31f55e..3ab4de68af34 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf @@ -20,11 +20,6 @@ variable "project_id" { description = "The ID of the Google Cloud project within which resources are provisioned" } -variable "region" { - type = string - description = "The Google Cloud Platform (GCP) region in which to provision resources" -} - variable "cloudbuild_service_account_id" { type = string description = "The ID of the cloud build service account responsible for provisioning Google Cloud resources" diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf index 0795a1e98f96..13134488e395 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf @@ -43,16 +43,30 @@ variable "cloudbuild_service_account_id" { default = "playground-cloudbuild-sa" } -variable "playground_environment_name" {} +variable "playground_environment_name" { + description = "Environment where to deploy Playground. Located in playground/terraform/environment/{environment_name}" +} -variable "playground_dns_name" {} +variable "playground_dns_name" { + description = "DNS record name for Playground website" +} -variable "playground_network_name" {} +variable "playground_network_name" { + description = "GCP VPC Network Name for Playground deployment" +} -variable "playground_gke_name" {} +variable "playground_gke_name" { + description = "Playground GKE Cluster name" +} -variable "state_bucket" {} +variable "state_bucket" { + description = "GCS bucket name for Beam Playground temp files and Terraform state" +} -variable "image_tag" {} +variable "image_tag" { + description = "The tag name for docker images of Playground containers" +} -variable "docker_repository_root" {} \ No newline at end of file +variable "docker_repository_root" { + description = "The name of GCP Artifact Registry Repository where Playground images will be saved to" +} \ No newline at end of file From 06f4e354b182dcc9aaf157aeb020aafe9b25f96d Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 22 Nov 2022 03:34:35 +0500 Subject: [PATCH 039/269] Updates related to apache beam PR #24144 --- .../cloudbuild/cloudbuild_pg_infra.yaml | 25 +++++--- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 13 ---- .../01.setup/services.tf | 2 +- .../02.builders/triggers.tf | 24 +++---- .../cloudbuild-manual-setup/README.md | 62 ++++++++++++------- 5 files changed, 67 insertions(+), 59 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 498ceb98abbb..bbf85226073f 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -20,25 +20,30 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: - 'build' - - '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' + - '--tag=us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' - '-f' - 'playground/infrastructure/cloudbuild/Dockerfile' - '.' # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' + - name: 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' entrypoint: "/bin/bash" args: - '-c' - | mkdir playground/terraform/environment/$_ENVIRONMENT_NAME - printf '%s\n' 'project_id = $PROJECT_ID' \ - 'network_name = $_NETWORK_NAME' \ - 'gke_name = $_GKE_NAME' \ - 'region = $LOCATION' \ - 'location = $LOCATION-a' \ - 'state_bucket = $_STATE_BUCKET' \ + printf '%s\n' \ + 'project_id = "$PROJECT_ID"' \ + 'network_name = "$_NETWORK_NAME"' \ + 'gke_name = "$_GKE_NAME"' \ + 'region = "$LOCATION"' \ + 'location = "$LOCATION-a"' \ + 'state_bucket = "$_STATE_BUCKET"' \ + 'bucket_examples_name = "$_STATE_BUCKET-examples"' \ > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars + printf \ + 'bucket = "$_STATE_BUCKET"'\ + > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' @@ -47,10 +52,10 @@ steps: # Pushing built image to GCP Artifact Registry repository images: - - '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' + - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' # This option enables writing logs to Cloud Logging options: logging: CLOUD_LOGGING_ONLY -timeout: 1800s \ No newline at end of file +timeout: 3600s \ No newline at end of file diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index d874709411ec..2a6b923e25d7 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -26,19 +26,6 @@ steps: -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Pdns-name="$_DNS_NAME"' ] -# Substitutions can be changed in GCP Cloud Build Triggers UI -substitutions: - # The name of your environment in beam/playground/terraform/environment/ - _ENVIRONMENT_NAME: "test" - # The name of your DNS for Playground - _DNS_NAME: "dev-playground.online" - # The name of your Docker tag - _TAG: "playground" - # The name of your GKE cluster (from terraform.tfvars) - _GKE_NAME: "playground-cluster" - # The name of docker repository root (from outputs of first trigger result) - _DOCKER_REPOSITORY_ROOT: "$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository" - # This option enables writing logs to Cloud Logging options: logging: CLOUD_LOGGING_ONLY diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf index 1821c1512ace..d44822e37ba5 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf @@ -17,7 +17,7 @@ // Provision the required Google Cloud services resource "google_project_service" "required_services" { - project = var.project_id + project = var.project_id for_each = toset([ "cloudresourcemanager", "cloudbuild", diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf index 248e26ad4e03..2bd724e787e3 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf @@ -38,11 +38,11 @@ resource "google_cloudbuild_trigger" "playground_infrastructure" { } substitutions = { - _ENVIRONMENT_NAME: var.playground_environment_name - _DNS_NAME: var.playground_dns_name - _NETWORK_NAME: var.playground_network_name - _GKE_NAME: var.playground_gke_name - _STATE_BUCKET: var.state_bucket + _ENVIRONMENT_NAME : var.playground_environment_name + _DNS_NAME : var.playground_dns_name + _NETWORK_NAME : var.playground_network_name + _GKE_NAME : var.playground_gke_name + _STATE_BUCKET : var.state_bucket } service_account = data.google_service_account.cloudbuild_sa.id @@ -67,13 +67,13 @@ resource "google_cloudbuild_trigger" "playground_to_gke" { } substitutions = { - _ENVIRONMENT_NAME: var.playground_environment_name - _DNS_NAME: var.playground_dns_name - _NETWORK_NAME: var.playground_network_name - _GKE_NAME: var.playground_gke_name - _STATE_BUCKET: var.state_bucket - _TAG: var.image_tag - _DOCKER_REPOSITORY_ROOT: var.docker_repository_root + _ENVIRONMENT_NAME : var.playground_environment_name + _DNS_NAME : var.playground_dns_name + _NETWORK_NAME : var.playground_network_name + _GKE_NAME : var.playground_gke_name + _STATE_BUCKET : var.state_bucket + _TAG : var.image_tag + _DOCKER_REPOSITORY_ROOT : var.docker_repository_root } service_account = data.google_service_account.cloudbuild_sa.id diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 2a822b6721c2..b58dc67e5a55 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -46,12 +46,26 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi #### To execute the module: 1. Set following environment variables: - - STATE_BUCKET - GCP Storage bucket name to save Terraform state - - GOOGLE_PROJECT - GCP Project ID - - GOOGLE_REGION - GCP region to save resources to + - `STATE_BUCKET`: GCP Storage bucket name to save Terraform state + - `GOOGLE_PROJECT`: GCP Project ID + - `GOOGLE_REGION`: GCP region to save resources to + - `ENVIRONMENT_NAME`: Environment where Playground will be deployed (located at playground/terraform/environment/environment_name) + - `DNS_NAME`: DNS for deployed Playground webpage + - `NETWORK_NAME`: GCP VPC Network Name for Playground deployment + - `GKE_NAME`: Playground GKE Cluster name + - `TAG`: Tag for Playground images + - `DOCKER_REPOSITORY_ROOT`: GCP Artifact Registry repository name to store Playground container images ```console - export STATE_BUCKET="state_bucket" GOOGLE_PROJECT="project_id" GOOGLE_REGION="region" + export STATE_BUCKET="state_bucket" \ + GOOGLE_PROJECT="project_id" \ + GOOGLE_REGION="region" \ + ENVIRONMENT_NAME="env_name" \ + DNS_NAME="dns_name" \ + NETWORK_NAME="network_name" \ + GKE_NAME="gke_name" \ + TAG="tag" \ + DOCKER_REPOSITORY_ROOT="us-central1-docker.pkg.dev/playground-repository" ``` **Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. @@ -93,35 +107,37 @@ cd ../02.builders # Run terraform scripts terraform init -backend-config="bucket=$STATE_BUCKET" -terraform apply -var "project_id=$GOOGLE_PROJECT" -var "region=$GOOGLE_REGION" + +terraform apply \ +-var "project_id=$GOOGLE_PROJECT" \ +-var "region=$GOOGLE_REGION" \ +-var "playground_environment_name=$ENVIRONMENT_NAME" \ +-var "playground_dns_name=$DNS_NAME" \ +-var "playground_network_name=$NETWORK_NAME" \ +-var "playground_gke_name=$GKE_NAME" \ +-var "state_bucket=$STATE_BUCKET" \ +-var "image_tag=$TAG" \ +-var "docker_repository_root=$DOCKER_REPOSITORY_ROOT" ``` ## 4. Run Cloud Build `Playground-infrastructure-trigger` to deploy Playground infrastructure -1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (Example: us-central1). +1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (In our example: us-central1). 2. Open Trigger: `Playground-infrastructure-trigger`. -3. Scroll down to `Advanced` - `Substitutions variables` and Click on `+ ADD VARIABLE`. -4. Assign values for the next variables: - - `_ENVIRONMENT_NAME`           # *Created "environment_name"* - - `_DNS_NAME`                         *# Your DNS record name* - - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name `logs_bucket`* -5. Click `Save` and Run the trigger `Playground-infrastructure-trigger`. - -6. Once Playground infrastructure has been deployed, please navigate to +3. Scroll down to `Source` - `Repository` to ensure that Apache Beam GitHub repository is connected. + - Click on drop-down menu and press `CONNECT NEW REPOSITORY` in case it was not automatically connected. +4. Click `Save` and Run the trigger `Playground-infrastructure-trigger`. + +5. Once Playground infrastructure has been deployed, please navigate to [Playground deployment README](https://github.com/apache/beam/tree/master/playground/terraform#deploy-playground-infrastructure) and execute step #2: - `Add following DNS A records for the discovered static IP address` + `Add following DNS A records for the discovered static IP address` expanding use of variable `DNS_NAME`. ## 5. Run Cloud Build `Playground-to-gke-trigger` to deploy Playground to GKE -1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (Example: us-central1). +1. Navigate to [GCP Console Cloud Build Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose the region (In our example: us-central1). 2. Open Trigger: `Playground-to-gke-trigger`. -3. Scroll down to `Advanced` - `Substitutions variables` and Click on `+ ADD VARIABLE`. -4. Assign values for the next variables: - - `_ENVIRONMENT_NAME`           # *Created "environment_name"* - - `_DNS_NAME`                         *# Your DNS record name* - - `_TAG`                                 *# Tag name for your Playground container images* - - `_GKE_NAME`                         *# Your GKE cluster name (configured in terraform.tfvars)* - - `_LOGS_BUCKET_NAME`           *# Your GCP logs bucket name `logs_bucket`* +3. Scroll down to `Source` - `Repository` to ensure that Apache Beam GitHub repository is connected. + - Click on drop-down menu and press `CONNECT NEW REPOSITORY` in case it was not automatically connected. 5. Click `Save` and Run the trigger `Playground-to-gke-trigger`. ## 6. Validate Playground deployment From 5fd0082cb284fe4e66a776918be06b5ffd1094ee Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 22 Nov 2022 16:01:23 +0500 Subject: [PATCH 040/269] Update iam.tf --- .../infrastructure/cloudbuild-manual-setup/01.setup/iam.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf index f961f32af719..3117134ffcc9 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf @@ -35,7 +35,8 @@ resource "google_project_iam_member" "cloud_build_roles" { "roles/iam.securityAdmin", "roles/iam.serviceAccountUser", "roles/datastore.indexAdmin", - "roles/storage.admin" + "roles/storage.admin", + "roles/logging.logWriter" ]) role = each.key member = "serviceAccount:${google_service_account.cloudbuild_service_account_id.email}" From c6fad411e5d8e35bb888a506523e6de05ed96dc4 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 22 Nov 2022 16:22:23 +0500 Subject: [PATCH 041/269] Update cloudbuild_pg_infra.yaml --- .../infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index bbf85226073f..3047bf9af562 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -45,15 +45,15 @@ steps: 'bucket = "$_STATE_BUCKET"'\ > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend +# Pushing built image to GCP Artifact Registry repository +images: + - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' + # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' entrypoint: "/bin/bash" args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] -# Pushing built image to GCP Artifact Registry repository -images: - - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' - # This option enables writing logs to Cloud Logging options: logging: CLOUD_LOGGING_ONLY From 4d9acb7ea03fd94d30518ce5d3988fceb77ca641 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 22 Nov 2022 16:23:25 +0500 Subject: [PATCH 042/269] Updating cloud build configs --- .../infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 10 +++++----- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 3047bf9af562..7541a5a2a3d5 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -45,15 +45,15 @@ steps: 'bucket = "$_STATE_BUCKET"'\ > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend -# Pushing built image to GCP Artifact Registry repository -images: - - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' - # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' + - name: 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' entrypoint: "/bin/bash" args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] +# Pushing built image to GCP Artifact Registry repository +images: + - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' + # This option enables writing logs to Cloud Logging options: logging: CLOUD_LOGGING_ONLY diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 2a6b923e25d7..dbddd62b216c 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -17,7 +17,7 @@ steps: # This stage uses pre-built Docker container to run gradle task for Playground pre-config and deployment to GKE - - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/playground-repository/builder:playground' + - name: 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' entrypoint: "/bin/bash" args: [ From 4338276113bfd1479f7b6826cc7cc951de62644b Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 22 Nov 2022 18:07:56 +0500 Subject: [PATCH 043/269] Update provider version to 4.0 --- .../cloudbuild-manual-setup/01.setup/terraform.tf | 2 +- .../cloudbuild-manual-setup/02.builders/terraform.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/terraform.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/terraform.tf index 3cd6610d6322..e7baa96ccd74 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/terraform.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/terraform.tf @@ -22,7 +22,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "~> 4.40.0" + version = "~> 4.0.0" } } } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf index e0da73116ef8..b428b681ed33 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf @@ -22,7 +22,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "~> 4.40.0" + version = "~> 4.0.0" } } } \ No newline at end of file From 9b8f9b73fb7a1e74bdec32f76145127fdb00681f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 22 Nov 2022 20:18:18 +0500 Subject: [PATCH 044/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index b58dc67e5a55..113616311edf 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -59,7 +59,7 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi ```console export STATE_BUCKET="state_bucket" \ GOOGLE_PROJECT="project_id" \ - GOOGLE_REGION="region" \ + GOOGLE_REGION="us-central1" \ ENVIRONMENT_NAME="env_name" \ DNS_NAME="dns_name" \ NETWORK_NAME="network_name" \ From dfb72cf1d121cf8652ff43935594809bc1b50f02 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 15:40:59 +0500 Subject: [PATCH 045/269] Updates with bash scripts for cloud builder --- .../cloudbuild/cloudbuild_pg_infra.yaml | 37 +------------------ .../infrastructure/cloudbuild/env_init.sh | 35 ++++++++++++++++++ .../infrastructure/cloudbuild/tfvars_init.sh | 15 ++++++++ 3 files changed, 52 insertions(+), 35 deletions(-) create mode 100644 playground/infrastructure/cloudbuild/env_init.sh create mode 100644 playground/infrastructure/cloudbuild/tfvars_init.sh diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 7541a5a2a3d5..7e304e284686 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -16,43 +16,10 @@ # under the License. steps: - # This stage builds the image from Dockerfile that will be used as environment to run Beam Playground deployment - - name: 'gcr.io/cloud-builders/docker' - args: - - 'build' - - '--tag=us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' - - '-f' - - 'playground/infrastructure/cloudbuild/Dockerfile' - - '.' - - # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - - name: 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' - entrypoint: "/bin/bash" - args: - - '-c' - - | - mkdir playground/terraform/environment/$_ENVIRONMENT_NAME - printf '%s\n' \ - 'project_id = "$PROJECT_ID"' \ - 'network_name = "$_NETWORK_NAME"' \ - 'gke_name = "$_GKE_NAME"' \ - 'region = "$LOCATION"' \ - 'location = "$LOCATION-a"' \ - 'state_bucket = "$_STATE_BUCKET"' \ - 'bucket_examples_name = "$_STATE_BUCKET-examples"' \ - > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars - printf \ - 'bucket = "$_STATE_BUCKET"'\ - > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend - # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - name: 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' - entrypoint: "/bin/bash" - args: ['-c', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] - -# Pushing built image to GCP Artifact Registry repository -images: - - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' + entrypoint: 'bash' + args: ['./env_init.sh', './tfvars_init.sh', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] # This option enables writing logs to Cloud Logging options: diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh new file mode 100644 index 000000000000..e8a90070d3b9 --- /dev/null +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +GO_VERSION=1.18 + +apt update >/dev/null + +# Install JRE +apt install -y default-jre && apt install -y apt-transport-https ca-certificates curl software-properties-common + +# Install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" +apt install -y docker-ce + +#Install Helm +curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh + +# Install Terraform +wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ +| tee /etc/apt/sources.list.d/hashicorp.list +apt update -y && apt install -y terraform + +# Install kubectl +apt-get install -y apt-transport-https ca-certificates gnupg +curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg +echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" \ +| tee /etc/apt/sources.list.d/kubernetes.list +apt-get update && apt install -y kubectl + +# Install golang +curl -O https://dl.google.com/go/go"$GO_VERSION".linux-amd64.tar.gz +tar -C /usr/local -xvf go"$GO_VERSION".linux-amd64.tar.gz \ No newline at end of file diff --git a/playground/infrastructure/cloudbuild/tfvars_init.sh b/playground/infrastructure/cloudbuild/tfvars_init.sh new file mode 100644 index 000000000000..c3fd2692cb40 --- /dev/null +++ b/playground/infrastructure/cloudbuild/tfvars_init.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +mkdir playground/terraform/environment/$_ENVIRONMENT_NAME +printf '%s\n' \ +'project_id = "$PROJECT_ID"' \ +'network_name = "$_NETWORK_NAME"' \ +'gke_name = "$_GKE_NAME"' \ +'region = "$LOCATION"' \ +'location = "$LOCATION-a"' \ +'state_bucket = "$_STATE_BUCKET"' \ +'bucket_examples_name = "$_STATE_BUCKET-examples"' \ +> playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars +printf \ +'bucket = "$_STATE_BUCKET"'\ +> playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend \ No newline at end of file From 1d2560eedee5eb8dcd130ae69a1f98ccd6d0e10f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 15:45:32 +0500 Subject: [PATCH 046/269] Update cloudbuild_pg_infra.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 7e304e284686..0a39137f4995 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -17,7 +17,7 @@ steps: # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - - name: 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' + - name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' entrypoint: 'bash' args: ['./env_init.sh', './tfvars_init.sh', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] From 4d2eaecaa5a52f2f8818e2c229943f23cae889f7 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 15:47:51 +0500 Subject: [PATCH 047/269] Update cloudbuild_pg_infra.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 0a39137f4995..adadbfc364e7 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -19,7 +19,9 @@ steps: # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' entrypoint: 'bash' - args: ['./env_init.sh', './tfvars_init.sh', './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] + args: ['./playground/infrastructure/cloudbuild/env_init.sh', + './playground/infrastructure/cloudbuild/tfvars_init.sh', + './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] # This option enables writing logs to Cloud Logging options: From 86355e7349496803e4a9ce6a918cc572e4b48da7 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 16:13:40 +0500 Subject: [PATCH 048/269] Updates of cloud build logic --- .../infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 9 ++++++--- playground/infrastructure/cloudbuild/env_init.sh | 7 +++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index adadbfc364e7..b9232ff1a015 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -19,9 +19,12 @@ steps: # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' entrypoint: 'bash' - args: ['./playground/infrastructure/cloudbuild/env_init.sh', - './playground/infrastructure/cloudbuild/tfvars_init.sh', - './gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME"'] + args: + - '-c' + - | + ./playground/infrastructure/cloudbuild/env_init.sh \ + && ./playground/infrastructure/cloudbuild/tfvars_init.sh \ + && ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging options: diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index e8a90070d3b9..0386eea37fe2 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -18,10 +18,9 @@ chmod 700 get_helm.sh ./get_helm.sh # Install Terraform -wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg -echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ -| tee /etc/apt/sources.list.d/hashicorp.list -apt update -y && apt install -y terraform +wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list +sudo apt update -y && sudo apt install -y terraform # Install kubectl apt-get install -y apt-transport-https ca-certificates gnupg From 3b6014e9bd0af2fff3e4f73a21a3a09ca143a4bb Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 16:17:49 +0500 Subject: [PATCH 049/269] Update env_init.sh --- playground/infrastructure/cloudbuild/env_init.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index 0386eea37fe2..6f6ee659f97d 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -2,15 +2,15 @@ GO_VERSION=1.18 -apt update >/dev/null +sudo apt update >/dev/null # Install JRE -apt install -y default-jre && apt install -y apt-transport-https ca-certificates curl software-properties-common +sudo apt install -y default-jre && sudo apt install -y apt-transport-https ca-certificates curl software-properties-common # Install Docker -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" -apt install -y docker-ce +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" +sudo apt install -y docker-ce #Install Helm curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 @@ -23,11 +23,11 @@ echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https:// sudo apt update -y && sudo apt install -y terraform # Install kubectl -apt-get install -y apt-transport-https ca-certificates gnupg +sudo apt-get install -y apt-transport-https ca-certificates gnupg curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" \ -| tee /etc/apt/sources.list.d/kubernetes.list -apt-get update && apt install -y kubectl +| sudo tee /etc/apt/sources.list.d/kubernetes.list +sudo apt-get update && sudo apt install -y kubectl # Install golang curl -O https://dl.google.com/go/go"$GO_VERSION".linux-amd64.tar.gz From 50cd0ccaaa99264cea811535825b88188eed1615 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 16:21:43 +0500 Subject: [PATCH 050/269] Update cloudbuild_pg_infra.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index b9232ff1a015..137130fa4a5b 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -22,7 +22,8 @@ steps: args: - '-c' - | - ./playground/infrastructure/cloudbuild/env_init.sh \ + chmod +x ./playground/infrastructure/cloudbuild/env_init.sh && ./playground/infrastructure/cloudbuild/tfvars_init.sh && \ + && ./playground/infrastructure/cloudbuild/env_init.sh \ && ./playground/infrastructure/cloudbuild/tfvars_init.sh \ && ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" From 090fa32c68250e3074a19bb344cd060f789901d1 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 16:24:04 +0500 Subject: [PATCH 051/269] Update cloudbuild_pg_infra.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 137130fa4a5b..db62405b01fd 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -22,7 +22,7 @@ steps: args: - '-c' - | - chmod +x ./playground/infrastructure/cloudbuild/env_init.sh && ./playground/infrastructure/cloudbuild/tfvars_init.sh && \ + chmod +x ./playground/infrastructure/cloudbuild/env_init.sh && ./playground/infrastructure/cloudbuild/tfvars_init.sh \ && ./playground/infrastructure/cloudbuild/env_init.sh \ && ./playground/infrastructure/cloudbuild/tfvars_init.sh \ && ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" From 5c5bce614e192e581c4354c44e5cee88d2cc6d80 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 16:27:41 +0500 Subject: [PATCH 052/269] Updates to cb bash scripts --- playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 3 +-- playground/infrastructure/cloudbuild/env_init.sh | 1 - playground/infrastructure/cloudbuild/tfvars_init.sh | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index db62405b01fd..b9232ff1a015 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -22,8 +22,7 @@ steps: args: - '-c' - | - chmod +x ./playground/infrastructure/cloudbuild/env_init.sh && ./playground/infrastructure/cloudbuild/tfvars_init.sh \ - && ./playground/infrastructure/cloudbuild/env_init.sh \ + ./playground/infrastructure/cloudbuild/env_init.sh \ && ./playground/infrastructure/cloudbuild/tfvars_init.sh \ && ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index 6f6ee659f97d..1564d83d766f 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -1,5 +1,4 @@ #!/bin/bash - GO_VERSION=1.18 sudo apt update >/dev/null diff --git a/playground/infrastructure/cloudbuild/tfvars_init.sh b/playground/infrastructure/cloudbuild/tfvars_init.sh index c3fd2692cb40..04b8e496378d 100644 --- a/playground/infrastructure/cloudbuild/tfvars_init.sh +++ b/playground/infrastructure/cloudbuild/tfvars_init.sh @@ -1,5 +1,4 @@ #!/bin/bash - mkdir playground/terraform/environment/$_ENVIRONMENT_NAME printf '%s\n' \ 'project_id = "$PROJECT_ID"' \ From 62cfef255b114da4f29267fcd041f425ab8609aa Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 16:30:51 +0500 Subject: [PATCH 053/269] Update cloudbuild_pg_infra.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index b9232ff1a015..e593ac61dfae 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -22,8 +22,8 @@ steps: args: - '-c' - | - ./playground/infrastructure/cloudbuild/env_init.sh \ - && ./playground/infrastructure/cloudbuild/tfvars_init.sh \ + playground/infrastructure/cloudbuild/env_init.sh \ + && playground/infrastructure/cloudbuild/tfvars_init.sh \ && ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging From 08becf63dbeca8a8a89e0e7f523a66c19c5cbc45 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 16:55:24 +0500 Subject: [PATCH 054/269] Update cloudbuild_pg_infra.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index e593ac61dfae..ec38baf26c40 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -18,9 +18,9 @@ steps: # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' - entrypoint: 'bash' + entrypoint: "/bin/bash" args: - - '-c' + - "-c" - | playground/infrastructure/cloudbuild/env_init.sh \ && playground/infrastructure/cloudbuild/tfvars_init.sh \ From 75352c65da97b0ddd8ebddf6400734f271cc7421 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 17:00:08 +0500 Subject: [PATCH 055/269] Shell scripts updates --- .../infrastructure/cloudbuild/env_init.sh | 40 +++++++++++++------ .../infrastructure/cloudbuild/tfvars_init.sh | 18 ++++++++- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index 1564d83d766f..c812fd98df06 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -1,15 +1,31 @@ -#!/bin/bash +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + GO_VERSION=1.18 -sudo apt update >/dev/null +apt update >/dev/null # Install JRE -sudo apt install -y default-jre && sudo apt install -y apt-transport-https ca-certificates curl software-properties-common +apt install -y default-jre && apt install -y apt-transport-https ca-certificates curl software-properties-common # Install Docker -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" -sudo apt install -y docker-ce +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" +apt install -y docker-ce #Install Helm curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 @@ -17,16 +33,16 @@ chmod 700 get_helm.sh ./get_helm.sh # Install Terraform -wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg -echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list -sudo apt update -y && sudo apt install -y terraform +wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/hashicorp.list +apt update -y && apt install -y terraform # Install kubectl -sudo apt-get install -y apt-transport-https ca-certificates gnupg +apt-get install -y apt-transport-https ca-certificates gnupg curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" \ -| sudo tee /etc/apt/sources.list.d/kubernetes.list -sudo apt-get update && sudo apt install -y kubectl +| tee /etc/apt/sources.list.d/kubernetes.list +apt-get update && apt install -y kubectl # Install golang curl -O https://dl.google.com/go/go"$GO_VERSION".linux-amd64.tar.gz diff --git a/playground/infrastructure/cloudbuild/tfvars_init.sh b/playground/infrastructure/cloudbuild/tfvars_init.sh index 04b8e496378d..1e077d154e66 100644 --- a/playground/infrastructure/cloudbuild/tfvars_init.sh +++ b/playground/infrastructure/cloudbuild/tfvars_init.sh @@ -1,4 +1,20 @@ -#!/bin/bash +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + mkdir playground/terraform/environment/$_ENVIRONMENT_NAME printf '%s\n' \ 'project_id = "$PROJECT_ID"' \ From 496356dc13a4b4d8a3252142cc534258e0c13dec Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 17:05:29 +0500 Subject: [PATCH 056/269] Update cloudbuild_pg_infra.yaml --- .../infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index ec38baf26c40..6ece464e872b 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -18,12 +18,12 @@ steps: # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' - entrypoint: "/bin/bash" + entrypoint: "bash" args: - "-c" - | - playground/infrastructure/cloudbuild/env_init.sh \ - && playground/infrastructure/cloudbuild/tfvars_init.sh \ + ./playground/infrastructure/cloudbuild/env_init.sh \ + && ./playground/infrastructure/cloudbuild/tfvars_init.sh \ && ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging From 11d1e50ebfe0040eb40e53dd350a4f93e07b0c69 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 17:07:43 +0500 Subject: [PATCH 057/269] Update cloudbuild_pg_infra.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 6ece464e872b..ddcdb72f16ff 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -22,8 +22,8 @@ steps: args: - "-c" - | - ./playground/infrastructure/cloudbuild/env_init.sh \ - && ./playground/infrastructure/cloudbuild/tfvars_init.sh \ + cd playground/infrastructure/cloudbuild && chmod +x env_init.sh && ./env_init.sh \ + && chmod +x tfvars_init.sh && ./tfvars_init.sh \ && ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging From ef6233d8f187881018ecc5da719c328664bf0c7e Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 17:15:06 +0500 Subject: [PATCH 058/269] Shell script updates cloud build --- .../cloudbuild/cloudbuild_pg_infra.yaml | 14 ++++++++- .../infrastructure/cloudbuild/env_init.sh | 8 ++--- .../infrastructure/cloudbuild/tfvars_init.sh | 30 ------------------- 3 files changed, 17 insertions(+), 35 deletions(-) delete mode 100644 playground/infrastructure/cloudbuild/tfvars_init.sh diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index ddcdb72f16ff..8dc19a6d4429 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -23,7 +23,19 @@ steps: - "-c" - | cd playground/infrastructure/cloudbuild && chmod +x env_init.sh && ./env_init.sh \ - && chmod +x tfvars_init.sh && ./tfvars_init.sh \ + mkdir playground/terraform/environment/$_ENVIRONMENT_NAME + printf '%s\n' \ + 'project_id = "$PROJECT_ID"' \ + 'network_name = "$_NETWORK_NAME"' \ + 'gke_name = "$_GKE_NAME"' \ + 'region = "$LOCATION"' \ + 'location = "$LOCATION-a"' \ + 'state_bucket = "$_STATE_BUCKET"' \ + 'bucket_examples_name = "$_STATE_BUCKET-examples"' \ + > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars + printf \ + 'bucket = "$_STATE_BUCKET"'\ + > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend && ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index c812fd98df06..67f2a05e4c3f 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -20,12 +20,12 @@ GO_VERSION=1.18 apt update >/dev/null # Install JRE -apt install -y default-jre && apt install -y apt-transport-https ca-certificates curl software-properties-common +apt install -y default-jre && apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg2 # Install Docker -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" -apt install -y docker-ce +curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" +apt update && apt install -y docker-ce #Install Helm curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 diff --git a/playground/infrastructure/cloudbuild/tfvars_init.sh b/playground/infrastructure/cloudbuild/tfvars_init.sh deleted file mode 100644 index 1e077d154e66..000000000000 --- a/playground/infrastructure/cloudbuild/tfvars_init.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -mkdir playground/terraform/environment/$_ENVIRONMENT_NAME -printf '%s\n' \ -'project_id = "$PROJECT_ID"' \ -'network_name = "$_NETWORK_NAME"' \ -'gke_name = "$_GKE_NAME"' \ -'region = "$LOCATION"' \ -'location = "$LOCATION-a"' \ -'state_bucket = "$_STATE_BUCKET"' \ -'bucket_examples_name = "$_STATE_BUCKET-examples"' \ -> playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars -printf \ -'bucket = "$_STATE_BUCKET"'\ -> playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend \ No newline at end of file From 16d6ba4b53375dc9407e47c3e4fc3b7b2019f6ab Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 17:24:24 +0500 Subject: [PATCH 059/269] Updates --- .../infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 7 ++++--- playground/infrastructure/cloudbuild/env_init.sh | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 8dc19a6d4429..cabb68716cce 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -18,11 +18,12 @@ steps: # This stage uses Docker container from image built in step above to run gradle task for Playground Infrastructure deployment - name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' - entrypoint: "bash" + entrypoint: "/bin/bash" args: - "-c" - | - cd playground/infrastructure/cloudbuild && chmod +x env_init.sh && ./env_init.sh \ + chmod +x playground/infrastructure/cloudbuild/env_init.sh \ + && ./playground/infrastructure/cloudbuild/env_init.sh mkdir playground/terraform/environment/$_ENVIRONMENT_NAME printf '%s\n' \ 'project_id = "$PROJECT_ID"' \ @@ -36,7 +37,7 @@ steps: printf \ 'bucket = "$_STATE_BUCKET"'\ > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend - && ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" + ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="$_ENVIRONMENT_NAME" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging options: diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index 67f2a05e4c3f..616c1fb7cc25 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -20,7 +20,7 @@ GO_VERSION=1.18 apt update >/dev/null # Install JRE -apt install -y default-jre && apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg2 +apt install -y default-jre && apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg2 wget # Install Docker curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - From debfb341e01a372b37c93f237cd3f4e0370c9d5c Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 17:39:20 +0500 Subject: [PATCH 060/269] Updates related to PR #24144 --- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 27 ++++++++++++++----- .../infrastructure/cloudbuild/env_init.sh | 2 +- .../cloudbuild-manual-setup/README.md | 4 +-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index dbddd62b216c..ec71ac00c8f9 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -17,14 +17,29 @@ steps: # This stage uses pre-built Docker container to run gradle task for Playground pre-config and deployment to GKE - - name: 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/builder:playground' + - name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' entrypoint: "/bin/bash" args: - [ - '-c', './gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" && - ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" - -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Pdns-name="$_DNS_NAME"' - ] + - "-c" + - | + chmod +x playground/infrastructure/cloudbuild/env_init.sh \ + && ./playground/infrastructure/cloudbuild/env_init.sh + mkdir playground/terraform/environment/$_ENVIRONMENT_NAME + printf '%s\n' \ + 'project_id = "$PROJECT_ID"' \ + 'network_name = "$_NETWORK_NAME"' \ + 'gke_name = "$_GKE_NAME"' \ + 'region = "$LOCATION"' \ + 'location = "$LOCATION-a"' \ + 'state_bucket = "$_STATE_BUCKET"' \ + 'bucket_examples_name = "$_STATE_BUCKET-examples"' \ + > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars + printf \ + 'bucket = "$_STATE_BUCKET"'\ + > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend + ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" + ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ + -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging options: diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index 616c1fb7cc25..3d67cfad81a5 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -46,4 +46,4 @@ apt-get update && apt install -y kubectl # Install golang curl -O https://dl.google.com/go/go"$GO_VERSION".linux-amd64.tar.gz -tar -C /usr/local -xvf go"$GO_VERSION".linux-amd64.tar.gz \ No newline at end of file +tar -C /usr/local -xvf go"$GO_VERSION".linux-amd64.tar.gz > /dev/null \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 113616311edf..bed83595813b 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -64,10 +64,10 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi DNS_NAME="dns_name" \ NETWORK_NAME="network_name" \ GKE_NAME="gke_name" \ - TAG="tag" \ + TAG="tag_name" \ DOCKER_REPOSITORY_ROOT="us-central1-docker.pkg.dev/playground-repository" ``` -**Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. +**Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. 2. Run commands: From 0ba28553c1e8e2ae0774185d5683e89ac92965d8 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 18:29:03 +0500 Subject: [PATCH 061/269] Update terraform.tf --- .../cloudbuild-manual-setup/02.builders/terraform.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf index b428b681ed33..e0da73116ef8 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf @@ -22,7 +22,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "~> 4.0.0" + version = "~> 4.40.0" } } } \ No newline at end of file From 69c922b2efccd28a47019a173a9487d28e0c96b2 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 18:50:08 +0500 Subject: [PATCH 062/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index bed83595813b..c6d50a7d9cec 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -65,7 +65,7 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi NETWORK_NAME="network_name" \ GKE_NAME="gke_name" \ TAG="tag_name" \ - DOCKER_REPOSITORY_ROOT="us-central1-docker.pkg.dev/playground-repository" + DOCKER_REPOSITORY_ROOT="us-central1-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" ``` **Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. From a4c6ab8b98c2fb7a5d7f4fe344a573510bc01d6b Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 19:17:46 +0500 Subject: [PATCH 063/269] Updates related to PR #24144 --- .../infrastructure/cloudbuild/Dockerfile | 56 ------------------- .../infrastructure/cloudbuild/env_init.sh | 19 ++++--- 2 files changed, 10 insertions(+), 65 deletions(-) delete mode 100644 playground/infrastructure/cloudbuild/Dockerfile diff --git a/playground/infrastructure/cloudbuild/Dockerfile b/playground/infrastructure/cloudbuild/Dockerfile deleted file mode 100644 index 419f40442fac..000000000000 --- a/playground/infrastructure/cloudbuild/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM gcr.io/google.com/cloudsdktool/google-cloud-cli - -ARG GO_VERSION=1.18 -ENV PATH="${PATH}:/usr/local/go/bin" - -RUN apt-get update >/dev/null -RUN apt-get -y install build-essential unzip - -# Install Docker -# See: https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script -RUN curl -fsSL https://get.docker.com -o get-docker.sh -RUN sh ./get-docker.sh >/dev/null - -# Install Go -# See: https://go.dev/doc/install -RUN curl -OL https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz -RUN sha256sum go$GO_VERSION.linux-amd64.tar.gz -RUN tar -C /usr/local -xvf go$GO_VERSION.linux-amd64.tar.gz -RUN rm go$GO_VERSION.linux-amd64.tar.gz -RUN go version - -#Install kubectl -RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ - && chmod +x ./kubectl \ - && mv ./kubectl /usr/local/bin/kubectl \ - && curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 \ - && chmod +x get_helm.sh && ./get_helm.sh > /dev/null - -#Install Terraform -RUN curl -OL https://releases.hashicorp.com/terraform/1.3.3/terraform_1.3.3_linux_amd64.zip -RUN unzip terraform_1.3.3_linux_amd64.zip -RUN mv terraform /usr/local/bin - -#Install GoLint -RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.50.1 - -# Install Java -RUN apt-get install openjdk-11-jdk -y >/dev/null -RUN java -version \ No newline at end of file diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index 3d67cfad81a5..b2ea0dcccb00 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -17,32 +17,33 @@ GO_VERSION=1.18 -apt update >/dev/null +apt update > /dev/null # Install JRE -apt install -y default-jre && apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg2 wget +apt install -y default-jre > /dev/null \ +&& apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg2 wget > /dev/null # Install Docker curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" -apt update && apt install -y docker-ce +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" > /dev/null +apt update > /dev/null && apt install -y docker-ce > /dev/null #Install Helm -curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 > /dev/null chmod 700 get_helm.sh ./get_helm.sh # Install Terraform wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg -echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/hashicorp.list -apt update -y && apt install -y terraform +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ +| tee /etc/apt/sources.list.d/hashicorp.list +apt update -y > /dev/null && apt install -y terraform > /dev/null # Install kubectl -apt-get install -y apt-transport-https ca-certificates gnupg curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" \ | tee /etc/apt/sources.list.d/kubernetes.list -apt-get update && apt install -y kubectl +apt update > /dev/null && apt install -y kubectl > /dev/null # Install golang curl -O https://dl.google.com/go/go"$GO_VERSION".linux-amd64.tar.gz From 6545170605995fcf75cb628dee1c640ecd90f864 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 19:29:14 +0500 Subject: [PATCH 064/269] Update cloudbuild_pg_to_gke.yaml --- .../infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index ec71ac00c8f9..ec4d3acb85c7 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -37,8 +37,8 @@ steps: printf \ 'bucket = "$_STATE_BUCKET"'\ > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend - ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" - ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ + ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" \ + && ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging From 65f0a222ce2c9dd4dcda12d36e1b44d542180787 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 19:30:25 +0500 Subject: [PATCH 065/269] Revert "Update cloudbuild_pg_to_gke.yaml" This reverts commit 6545170605995fcf75cb628dee1c640ecd90f864. --- .../infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index ec4d3acb85c7..ec71ac00c8f9 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -37,8 +37,8 @@ steps: printf \ 'bucket = "$_STATE_BUCKET"'\ > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend - ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" \ - && ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ + ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" + ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging From 8888ff482bdaf76d9ea9542782cc08880044f1cc Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 19:30:58 +0500 Subject: [PATCH 066/269] Update triggers.tf --- .../cloudbuild-manual-setup/02.builders/triggers.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf index 2bd724e787e3..c9a0d750da1e 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf @@ -62,7 +62,7 @@ resource "google_cloudbuild_trigger" "playground_to_gke" { } git_file_source { - path = "playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml" + path = "playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml" repo_type = "GITHUB" } From 7b2ecd925eab2441e7aabde3100529c56026bf47 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 21:14:56 +0500 Subject: [PATCH 067/269] Updates with new variables for region/location #24144 --- .../infrastructure/cloudbuild/cloudbuild_pg_infra.yaml | 4 ++-- .../infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 4 ++-- .../cloudbuild-manual-setup/02.builders/triggers.tf | 4 ++++ .../cloudbuild-manual-setup/02.builders/variables.tf | 8 ++++++++ .../infrastructure/cloudbuild-manual-setup/README.md | 8 +++++++- 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index cabb68716cce..44371c90a838 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -29,8 +29,8 @@ steps: 'project_id = "$PROJECT_ID"' \ 'network_name = "$_NETWORK_NAME"' \ 'gke_name = "$_GKE_NAME"' \ - 'region = "$LOCATION"' \ - 'location = "$LOCATION-a"' \ + 'region = "$_PLAYGROUND_REGION"' \ + 'location = "$_PLAYGROUND_LOCATION"' \ 'state_bucket = "$_STATE_BUCKET"' \ 'bucket_examples_name = "$_STATE_BUCKET-examples"' \ > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index ec71ac00c8f9..8260ad573716 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -29,8 +29,8 @@ steps: 'project_id = "$PROJECT_ID"' \ 'network_name = "$_NETWORK_NAME"' \ 'gke_name = "$_GKE_NAME"' \ - 'region = "$LOCATION"' \ - 'location = "$LOCATION-a"' \ + 'region = "$_PLAYGROUND_REGION"' \ + 'location = "$_PLAYGROUND_LOCATION"' \ 'state_bucket = "$_STATE_BUCKET"' \ 'bucket_examples_name = "$_STATE_BUCKET-examples"' \ > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf index c9a0d750da1e..edf617f66d16 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf @@ -38,6 +38,8 @@ resource "google_cloudbuild_trigger" "playground_infrastructure" { } substitutions = { + _PLAYGROUND_REGION : var.playground_region + _PLAYGROUND_LOCATION : var.playground_location _ENVIRONMENT_NAME : var.playground_environment_name _DNS_NAME : var.playground_dns_name _NETWORK_NAME : var.playground_network_name @@ -67,6 +69,8 @@ resource "google_cloudbuild_trigger" "playground_to_gke" { } substitutions = { + _PLAYGROUND_REGION : var.playground_region + _PLAYGROUND_LOCATION : var.playground_location _ENVIRONMENT_NAME : var.playground_environment_name _DNS_NAME : var.playground_dns_name _NETWORK_NAME : var.playground_network_name diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf index 13134488e395..5389649d9808 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf @@ -69,4 +69,12 @@ variable "image_tag" { variable "docker_repository_root" { description = "The name of GCP Artifact Registry Repository where Playground images will be saved to" +} + +variable "playground_region" { + description = "Region (us-central1) where playground infrastructure will be deployed" +} + +variable "playground_location" { + description = "Location (us-central1-b) where playground infrastructure will be deployed" } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index c6d50a7d9cec..4fb23c3fde04 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -48,7 +48,9 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi 1. Set following environment variables: - `STATE_BUCKET`: GCP Storage bucket name to save Terraform state - `GOOGLE_PROJECT`: GCP Project ID - - `GOOGLE_REGION`: GCP region to save resources to + - `GOOGLE_REGION`: GCP region to save triggers to + - `PLAYGROUND_REGION`: GCP Region (us-central1) where playground infrastructure will be deployed + - `PLAYGROUND_LOCATION`: GCP Location (us-central1-b) where playground infrastructure will be deployed - `ENVIRONMENT_NAME`: Environment where Playground will be deployed (located at playground/terraform/environment/environment_name) - `DNS_NAME`: DNS for deployed Playground webpage - `NETWORK_NAME`: GCP VPC Network Name for Playground deployment @@ -60,6 +62,8 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi export STATE_BUCKET="state_bucket" \ GOOGLE_PROJECT="project_id" \ GOOGLE_REGION="us-central1" \ + PLAYGROUND_REGION="us-central1" \ + PLAYGROUND_LOCATION="us-central1-b" \ ENVIRONMENT_NAME="env_name" \ DNS_NAME="dns_name" \ NETWORK_NAME="network_name" \ @@ -111,6 +115,8 @@ terraform init -backend-config="bucket=$STATE_BUCKET" terraform apply \ -var "project_id=$GOOGLE_PROJECT" \ -var "region=$GOOGLE_REGION" \ +-var "playground_region=$PLAYGROUND_REGION" \ +- var "playground_location=$PLAYGROUND_LOCATION" \ -var "playground_environment_name=$ENVIRONMENT_NAME" \ -var "playground_dns_name=$DNS_NAME" \ -var "playground_network_name=$NETWORK_NAME" \ From 0971f39f905ec29ebe75423c2d0bd56cc0d9f9c0 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 23 Nov 2022 23:35:22 +0500 Subject: [PATCH 068/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 4fb23c3fde04..f3ccff9efc7c 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -69,7 +69,8 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi NETWORK_NAME="network_name" \ GKE_NAME="gke_name" \ TAG="tag_name" \ - DOCKER_REPOSITORY_ROOT="us-central1-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" + DOCKER_REPOSITORY_ROOT="$LOCATION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" + ``` **Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. From 035bc5b72346c40cb7b711a4985a4a7d8072c309 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 16:36:38 +0500 Subject: [PATCH 069/269] Updated for SDK_TAG --- .../infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- .../cloudbuild-manual-setup/02.builders/triggers.tf | 1 + .../cloudbuild-manual-setup/02.builders/variables.tf | 4 ++++ .../infrastructure/cloudbuild-manual-setup/README.md | 5 +++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 8260ad573716..0faf04b05a9c 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -39,7 +39,7 @@ steps: > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ - -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Pdns-name="$_DNS_NAME" + -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk_tag="_SDK_TAG" ""-Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging options: diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf index edf617f66d16..343e168dc383 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf @@ -78,6 +78,7 @@ resource "google_cloudbuild_trigger" "playground_to_gke" { _STATE_BUCKET : var.state_bucket _TAG : var.image_tag _DOCKER_REPOSITORY_ROOT : var.docker_repository_root + _SDK_TAG : var.sdk_tag } service_account = data.google_service_account.cloudbuild_sa.id diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf index 5389649d9808..56ee9d3a84a7 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf @@ -77,4 +77,8 @@ variable "playground_region" { variable "playground_location" { description = "Location (us-central1-b) where playground infrastructure will be deployed" +} + +variable "sdk_tag" { + description = "Apache Beam go python image sdk tag" } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index f3ccff9efc7c..7a56c1ea27fb 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -69,8 +69,8 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi NETWORK_NAME="network_name" \ GKE_NAME="gke_name" \ TAG="tag_name" \ + SDK_TAG="2.43.0" \ DOCKER_REPOSITORY_ROOT="$LOCATION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" - ``` **Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. @@ -117,13 +117,14 @@ terraform apply \ -var "project_id=$GOOGLE_PROJECT" \ -var "region=$GOOGLE_REGION" \ -var "playground_region=$PLAYGROUND_REGION" \ -- var "playground_location=$PLAYGROUND_LOCATION" \ +-var "playground_location=$PLAYGROUND_LOCATION" \ -var "playground_environment_name=$ENVIRONMENT_NAME" \ -var "playground_dns_name=$DNS_NAME" \ -var "playground_network_name=$NETWORK_NAME" \ -var "playground_gke_name=$GKE_NAME" \ -var "state_bucket=$STATE_BUCKET" \ -var "image_tag=$TAG" \ +-var "sdk_tag=$SDK_TAG" \ -var "docker_repository_root=$DOCKER_REPOSITORY_ROOT" ``` From c99684460925e9bfacd027ad895e55c6f90b4826 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 16:46:38 +0500 Subject: [PATCH 070/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 7a56c1ea27fb..254e34fcb073 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -56,6 +56,7 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi - `NETWORK_NAME`: GCP VPC Network Name for Playground deployment - `GKE_NAME`: Playground GKE Cluster name - `TAG`: Tag for Playground images + - `SDK_TAG`: Tag for SDK images of Apache Beam Go and Python (typically last stable 2.42.0, 2.43.0) - `DOCKER_REPOSITORY_ROOT`: GCP Artifact Registry repository name to store Playground container images ```console From 385096109407e24bbb9330073827b37936aed16b Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 16:54:39 +0500 Subject: [PATCH 071/269] Update cloudbuild_pg_to_gke.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 0faf04b05a9c..9ddd5b5c926e 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -39,7 +39,7 @@ steps: > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ - -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk_tag="_SDK_TAG" ""-Pdns-name="$_DNS_NAME" + -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk_tag="$_SDK_TAG" ""-Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging options: From 912986c231177d37dc66073db231ba7a744c8246 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 17:11:40 +0500 Subject: [PATCH 072/269] Update cloudbuild_pg_to_gke.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 9ddd5b5c926e..d7710f4a3d13 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -39,7 +39,7 @@ steps: > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ - -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk_tag="$_SDK_TAG" ""-Pdns-name="$_DNS_NAME" + -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk_tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging options: From 5a93aee0efcf1fadbf80541912ba57dd1db62500 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 17:12:39 +0500 Subject: [PATCH 073/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 254e34fcb073..fd660a745c67 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -71,7 +71,7 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi GKE_NAME="gke_name" \ TAG="tag_name" \ SDK_TAG="2.43.0" \ - DOCKER_REPOSITORY_ROOT="$LOCATION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" + DOCKER_REPOSITORY_ROOT="$_PLAYGROUND_REGION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" ``` **Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. From c7551570b7f60d009210a6bd42e723cf81f56b52 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 18:18:47 +0500 Subject: [PATCH 074/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index dca9a8dc5024..af6bb44b60d1 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -478,23 +478,23 @@ helm { task ("gkebackend") { group = "deploy" val init = tasks.getByName("terraformInit") - val apply = tasks.getByName("terraformApplyInf") - val indexcreate = tasks.getByName("indexcreate") + // val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") + val back = tasks.getByName("pushBack") val front = tasks.getByName("pushFront") - val push = tasks.getByName("pushBack") + val indexcreate = tasks.getByName("indexcreate") val helm = tasks.getByName("helmInstallPlayground") dependsOn(init) - dependsOn(apply) + // dependsOn(apply) dependsOn(takeConfig) - dependsOn(push) + dependsOn(back) dependsOn(front) dependsOn(indexcreate) dependsOn(helm) - apply.mustRunAfter(init) - takeConfig.mustRunAfter(apply) - push.mustRunAfter(takeConfig) - front.mustRunAfter(push) + // apply.mustRunAfter(init) + takeConfig.mustRunAfter(init) + back.mustRunAfter(takeConfig) + front.mustRunAfter(back) indexcreate.mustRunAfter(front) helm.mustRunAfter(indexcreate) } From b70afde6a15e2ba2396c4b3154c281e1748e3e6b Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 18:43:30 +0500 Subject: [PATCH 075/269] Updates --- playground/backend/containers/python/Dockerfile | 2 +- playground/terraform/build.gradle.kts | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/playground/backend/containers/python/Dockerfile b/playground/backend/containers/python/Dockerfile index cf6d543bb303..b4d1d18bc973 100644 --- a/playground/backend/containers/python/Dockerfile +++ b/playground/backend/containers/python/Dockerfile @@ -17,7 +17,7 @@ ############################################################################### ARG GO_BASE_IMAGE=golang:1.18-bullseye ARG SDK_TAG -ARG BASE_IMAGE=apache/beam_python3.7_sdk:$SDK_TAG +ARG BASE_IMAGE=apache/beam_python3.7_sdk:2.43.0 FROM $GO_BASE_IMAGE AS build # Setup Go Environment diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index af6bb44b60d1..f0027017fe08 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -475,6 +475,15 @@ helm { } } } + +task ("echo") { + doLast{ + exec { + executable("cat") + args("playground/infrastructure/helm-playground/values.yaml") + } + } +} task ("gkebackend") { group = "deploy" val init = tasks.getByName("terraformInit") From a1d09373ea780d1f855584a51506182dbed16697 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 18:50:46 +0500 Subject: [PATCH 076/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index f0027017fe08..083036115598 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -489,6 +489,7 @@ task ("gkebackend") { val init = tasks.getByName("terraformInit") // val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") + val echo = tasks.getByName("echo") val back = tasks.getByName("pushBack") val front = tasks.getByName("pushFront") val indexcreate = tasks.getByName("indexcreate") @@ -496,13 +497,15 @@ task ("gkebackend") { dependsOn(init) // dependsOn(apply) dependsOn(takeConfig) + dependsOn(echo) dependsOn(back) dependsOn(front) dependsOn(indexcreate) dependsOn(helm) // apply.mustRunAfter(init) takeConfig.mustRunAfter(init) - back.mustRunAfter(takeConfig) + echo.mustRunAfter(takeConfig) + back.mustRunAfter(echo) front.mustRunAfter(back) indexcreate.mustRunAfter(front) helm.mustRunAfter(indexcreate) From d354909774d8986fc760d9df2bafba771dbf8f08 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 18:53:32 +0500 Subject: [PATCH 077/269] Update cloudbuild_pg_to_gke.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index d7710f4a3d13..f3fba2cb3782 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -39,7 +39,7 @@ steps: > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ - -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk_tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" + -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK-TAG" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging options: From c5d6018d20a18bc5bfe1d5b8b2cc366ee0139542 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 18:53:50 +0500 Subject: [PATCH 078/269] Update Dockerfile --- playground/backend/containers/python/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/backend/containers/python/Dockerfile b/playground/backend/containers/python/Dockerfile index b4d1d18bc973..cf6d543bb303 100644 --- a/playground/backend/containers/python/Dockerfile +++ b/playground/backend/containers/python/Dockerfile @@ -17,7 +17,7 @@ ############################################################################### ARG GO_BASE_IMAGE=golang:1.18-bullseye ARG SDK_TAG -ARG BASE_IMAGE=apache/beam_python3.7_sdk:2.43.0 +ARG BASE_IMAGE=apache/beam_python3.7_sdk:$SDK_TAG FROM $GO_BASE_IMAGE AS build # Setup Go Environment From 8e27e02cf5e3d7b087f608e31d3f99758886bc52 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 18:54:48 +0500 Subject: [PATCH 079/269] Update cloudbuild_pg_to_gke.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index f3fba2cb3782..7c8759f160dd 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -39,7 +39,7 @@ steps: > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ - -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK-TAG" -Pdns-name="$_DNS_NAME" + -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging options: From 63bfc182fef0189d04684aa0add9d3029a7f4ed9 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 19:09:16 +0500 Subject: [PATCH 080/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 083036115598..0bf870c3041d 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -480,7 +480,7 @@ task ("echo") { doLast{ exec { executable("cat") - args("playground/infrastructure/helm-playground/values.yaml") + args("../infrastructure/helm-playground/values.yaml") } } } From 95f0c9a9a25eb46f6a1e8d8f7687969e0e606c1a Mon Sep 17 00:00:00 2001 From: oborysevych Date: Thu, 24 Nov 2022 17:36:12 +0200 Subject: [PATCH 081/269] test faster without back --- playground/terraform/build.gradle.kts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 0bf870c3041d..ac67b1223de3 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -490,7 +490,7 @@ task ("gkebackend") { // val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") val echo = tasks.getByName("echo") - val back = tasks.getByName("pushBack") + //val back = tasks.getByName("pushBack") val front = tasks.getByName("pushFront") val indexcreate = tasks.getByName("indexcreate") val helm = tasks.getByName("helmInstallPlayground") @@ -498,15 +498,17 @@ task ("gkebackend") { // dependsOn(apply) dependsOn(takeConfig) dependsOn(echo) - dependsOn(back) + //dependsOn(back) dependsOn(front) dependsOn(indexcreate) + + dependsOn(helm) // apply.mustRunAfter(init) takeConfig.mustRunAfter(init) echo.mustRunAfter(takeConfig) - back.mustRunAfter(echo) - front.mustRunAfter(back) + //back.mustRunAfter(echo) + front.mustRunAfter(echo) indexcreate.mustRunAfter(front) helm.mustRunAfter(indexcreate) } From 96b4c3e23d87cd493ae64af6b5477608be88bbe9 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 21:04:49 +0500 Subject: [PATCH 082/269] Update env_init.sh --- playground/infrastructure/cloudbuild/env_init.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index b2ea0dcccb00..c1d903898188 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -30,13 +30,12 @@ apt update > /dev/null && apt install -y docker-ce > /dev/null #Install Helm curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 > /dev/null -chmod 700 get_helm.sh -./get_helm.sh +chmod 700 get_helm.sh && chmod +x get_helm.sh && ./get_helm.sh > /dev/null # Install Terraform -wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg +wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ -| tee /etc/apt/sources.list.d/hashicorp.list +| tee /etc/apt/sources.list.d/hashicorp.list > dev/null apt update -y > /dev/null && apt install -y terraform > /dev/null # Install kubectl From 4f8ba9a6ee1682412ff3e585320a3e4ebdbab5b0 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 21:47:39 +0500 Subject: [PATCH 083/269] Updates to cloud build --- .../infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 3 ++- playground/infrastructure/cloudbuild/env_init.sh | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 7c8759f160dd..c8c1297f0881 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -36,7 +36,8 @@ steps: > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars printf \ 'bucket = "$_STATE_BUCKET"'\ - > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend + > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend + terraform version ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index c1d903898188..6c8f899724f3 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -33,9 +33,9 @@ curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts chmod 700 get_helm.sh && chmod +x get_helm.sh && ./get_helm.sh > /dev/null # Install Terraform -wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null +wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ -| tee /etc/apt/sources.list.d/hashicorp.list > dev/null +| tee /etc/apt/sources.list.d/hashicorp.list apt update -y > /dev/null && apt install -y terraform > /dev/null # Install kubectl From 54bba59e1e64a098ef91be55e11801ee086f0699 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 22:03:11 +0500 Subject: [PATCH 084/269] Update cloudbuild_pg_to_gke.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index c8c1297f0881..84224c0f2052 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -38,6 +38,7 @@ steps: 'bucket = "$_STATE_BUCKET"'\ > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend terraform version + helm version ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" From b4c0e7c2b5b18c632155838df45b72e050d19a3e Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 22:09:08 +0500 Subject: [PATCH 085/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index ac67b1223de3..b63e1469d74d 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -41,6 +41,7 @@ val licenseText = "############################################################# plugins { id("com.pswidersk.terraform-plugin") version "1.0.0" id("org.unbroken-dome.helm") version "1.7.0" + id("org.unbroken-dome.helm-commands") version "1.7.0" id("org.unbroken-dome.helm-releases") version "1.7.0" } From 2041959004b70878522163a2ae357b98bebd79e2 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 24 Nov 2022 22:39:21 +0500 Subject: [PATCH 086/269] Updates cloud build --- .../infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- playground/terraform/build.gradle.kts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 84224c0f2052..e29330bcf129 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -47,4 +47,4 @@ steps: options: logging: CLOUD_LOGGING_ONLY -timeout: 3600s \ No newline at end of file +timeout: 7600s \ No newline at end of file diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index b63e1469d74d..ebdaa03f7fc5 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -491,7 +491,7 @@ task ("gkebackend") { // val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") val echo = tasks.getByName("echo") - //val back = tasks.getByName("pushBack") + val back = tasks.getByName("pushBack") val front = tasks.getByName("pushFront") val indexcreate = tasks.getByName("indexcreate") val helm = tasks.getByName("helmInstallPlayground") @@ -499,7 +499,7 @@ task ("gkebackend") { // dependsOn(apply) dependsOn(takeConfig) dependsOn(echo) - //dependsOn(back) + dependsOn(back) dependsOn(front) dependsOn(indexcreate) @@ -508,7 +508,7 @@ task ("gkebackend") { // apply.mustRunAfter(init) takeConfig.mustRunAfter(init) echo.mustRunAfter(takeConfig) - //back.mustRunAfter(echo) + back.mustRunAfter(echo) front.mustRunAfter(echo) indexcreate.mustRunAfter(front) helm.mustRunAfter(indexcreate) From b5b97bcd014bb5b93bdabdbb2daf683a3edf9144 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 00:14:23 +0500 Subject: [PATCH 087/269] Update cloudbuild_pg_to_gke.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index e29330bcf129..11032723cc1d 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -40,7 +40,7 @@ steps: terraform version helm version ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" - ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ + ./gradlew playground:terraform:gkebackend --stacktrace --debug -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging From b3a54730402dc61c505427f51315eb817f8e65b1 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 00:44:13 +0500 Subject: [PATCH 088/269] Update env_init.sh --- .../infrastructure/cloudbuild/env_init.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index 6c8f899724f3..ca167ca97e3f 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -17,11 +17,10 @@ GO_VERSION=1.18 -apt update > /dev/null +apt-get update > /dev/null # Install JRE -apt install -y default-jre > /dev/null \ -&& apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg2 wget > /dev/null +apt-get install -y build-essential unzip apt-transport-https ca-certificates curl software-properties-common gnupg2 wget > /dev/null # Install Docker curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - @@ -30,7 +29,7 @@ apt update > /dev/null && apt install -y docker-ce > /dev/null #Install Helm curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 > /dev/null -chmod 700 get_helm.sh && chmod +x get_helm.sh && ./get_helm.sh > /dev/null +chmod +x get_helm.sh && ./get_helm.sh > /dev/null # Install Terraform wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg @@ -39,11 +38,11 @@ echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https:// apt update -y > /dev/null && apt install -y terraform > /dev/null # Install kubectl -curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg -echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" \ -| tee /etc/apt/sources.list.d/kubernetes.list -apt update > /dev/null && apt install -y kubectl > /dev/null +curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ +&& chmod +x ./kubectl \ +&& mv ./kubectl /usr/local/bin/kubectl -# Install golang +# Install golang & jdk curl -O https://dl.google.com/go/go"$GO_VERSION".linux-amd64.tar.gz -tar -C /usr/local -xvf go"$GO_VERSION".linux-amd64.tar.gz > /dev/null \ No newline at end of file +tar -C /usr/local -xvf go"$GO_VERSION".linux-amd64.tar.gz > /dev/null +apt-get install openjdk-11-jdk -y > /dev/null \ No newline at end of file From d9db81a8fda491caf883759eb989e600ec60b186 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 01:10:54 +0500 Subject: [PATCH 089/269] Updates --- .../cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- playground/terraform/build.gradle.kts | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 11032723cc1d..e29330bcf129 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -40,7 +40,7 @@ steps: terraform version helm version ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" - ./gradlew playground:terraform:gkebackend --stacktrace --debug -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ + ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" # This option enables writing logs to Cloud Logging diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index ebdaa03f7fc5..38fe7c4ff636 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -488,7 +488,7 @@ task ("echo") { task ("gkebackend") { group = "deploy" val init = tasks.getByName("terraformInit") - // val apply = tasks.getByName("terraformApplyInf") + val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") val echo = tasks.getByName("echo") val back = tasks.getByName("pushBack") @@ -496,20 +496,18 @@ task ("gkebackend") { val indexcreate = tasks.getByName("indexcreate") val helm = tasks.getByName("helmInstallPlayground") dependsOn(init) - // dependsOn(apply) + dependsOn(apply) dependsOn(takeConfig) dependsOn(echo) dependsOn(back) dependsOn(front) dependsOn(indexcreate) - - dependsOn(helm) - // apply.mustRunAfter(init) - takeConfig.mustRunAfter(init) + apply.mustRunAfter(init) + takeConfig.mustRunAfter(apply) echo.mustRunAfter(takeConfig) back.mustRunAfter(echo) - front.mustRunAfter(echo) + front.mustRunAfter(back) indexcreate.mustRunAfter(front) helm.mustRunAfter(indexcreate) } From 871f149b6f427e87e6c4fadaef2a940b152b8235 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 14:18:54 +0500 Subject: [PATCH 090/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 38fe7c4ff636..a01c9e1e42b2 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -488,7 +488,7 @@ task ("echo") { task ("gkebackend") { group = "deploy" val init = tasks.getByName("terraformInit") - val apply = tasks.getByName("terraformApplyInf") +// val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") val echo = tasks.getByName("echo") val back = tasks.getByName("pushBack") @@ -496,15 +496,15 @@ task ("gkebackend") { val indexcreate = tasks.getByName("indexcreate") val helm = tasks.getByName("helmInstallPlayground") dependsOn(init) - dependsOn(apply) +// dependsOn(apply) dependsOn(takeConfig) dependsOn(echo) dependsOn(back) dependsOn(front) dependsOn(indexcreate) dependsOn(helm) - apply.mustRunAfter(init) - takeConfig.mustRunAfter(apply) +// apply.mustRunAfter(init) + takeConfig.mustRunAfter(init) echo.mustRunAfter(takeConfig) back.mustRunAfter(echo) front.mustRunAfter(back) From 55f47b808d8a3085b2931fc94d1c6589b40c41f8 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 14:52:06 +0500 Subject: [PATCH 091/269] Updates --- playground/infrastructure/cloudbuild/env_init.sh | 12 ++++++++---- playground/terraform/build.gradle.kts | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index ca167ca97e3f..a7179312bd5f 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -32,10 +32,14 @@ curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts chmod +x get_helm.sh && ./get_helm.sh > /dev/null # Install Terraform -wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg -echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ -| tee /etc/apt/sources.list.d/hashicorp.list -apt update -y > /dev/null && apt install -y terraform > /dev/null +#wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg +#echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ +#| tee /etc/apt/sources.list.d/hashicorp.list +#apt update -y > /dev/null && apt install -y terraform > /dev/null + +curl -OL https://releases.hashicorp.com/terraform/1.0.9/terraform_1.0.9_linux_amd64.zip +unzip terraform_1.0.9_linux_amd64.zip +mv terraform /usr/local/bin # Install kubectl curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index a01c9e1e42b2..57a84e8ea997 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -43,6 +43,7 @@ plugins { id("org.unbroken-dome.helm") version "1.7.0" id("org.unbroken-dome.helm-commands") version "1.7.0" id("org.unbroken-dome.helm-releases") version "1.7.0" + id("org.unbroken-dome.helm-publish") version "1.7.0" } terraformPlugin { From f131d691b1256f5b288d8bdbffc4627ef0ce5338 Mon Sep 17 00:00:00 2001 From: oborysevych Date: Fri, 25 Nov 2022 12:56:04 +0200 Subject: [PATCH 092/269] trying to set run helm verbose --- playground/build.gradle.kts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playground/build.gradle.kts b/playground/build.gradle.kts index b458ef1f7510..b8134b875056 100644 --- a/playground/build.gradle.kts +++ b/playground/build.gradle.kts @@ -18,6 +18,11 @@ description = "Apache Beam :: Playground" +subprojects { + // Use verbose logging on all Helm commands + helm.extraArgs.addAll("-v", "1") +} + // generate protobuf client and server stubs task("generateProto") { group = "build" From 5a123c0638a1a1f9d293f28397f9dbd7d45cd9f8 Mon Sep 17 00:00:00 2001 From: oborysevych Date: Fri, 25 Nov 2022 13:14:12 +0200 Subject: [PATCH 093/269] remove unneeded task temporarily --- playground/terraform/build.gradle.kts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 57a84e8ea997..9e43cb7bead6 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -492,23 +492,23 @@ task ("gkebackend") { // val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") val echo = tasks.getByName("echo") - val back = tasks.getByName("pushBack") + //val back = tasks.getByName("pushBack") val front = tasks.getByName("pushFront") - val indexcreate = tasks.getByName("indexcreate") - val helm = tasks.getByName("helmInstallPlayground") + //val indexcreate = tasks.getByName("indexcreate") + //val helm = tasks.getByName("helmInstallPlayground") dependsOn(init) // dependsOn(apply) dependsOn(takeConfig) dependsOn(echo) - dependsOn(back) + //dependsOn(back) dependsOn(front) - dependsOn(indexcreate) - dependsOn(helm) + //dependsOn(indexcreate) + //dependsOn(helm) // apply.mustRunAfter(init) takeConfig.mustRunAfter(init) echo.mustRunAfter(takeConfig) - back.mustRunAfter(echo) - front.mustRunAfter(back) - indexcreate.mustRunAfter(front) - helm.mustRunAfter(indexcreate) + //back.mustRunAfter(echo) + front.mustRunAfter(takeConfig) + //indexcreate.mustRunAfter(front) + //helm.mustRunAfter(indexcreate) } From ac50f55a1d9ef319e4e38b6db49bac6a7b6e0b62 Mon Sep 17 00:00:00 2001 From: oborysevych Date: Fri, 25 Nov 2022 13:15:19 +0200 Subject: [PATCH 094/269] remove extraargs --- playground/build.gradle.kts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playground/build.gradle.kts b/playground/build.gradle.kts index b8134b875056..01ff69ae43c8 100644 --- a/playground/build.gradle.kts +++ b/playground/build.gradle.kts @@ -18,10 +18,10 @@ description = "Apache Beam :: Playground" -subprojects { - // Use verbose logging on all Helm commands - helm.extraArgs.addAll("-v", "1") -} +// subprojects { +// // Use verbose logging on all Helm commands +// helm.extraArgs.addAll("-v", "1") +// } // generate protobuf client and server stubs task("generateProto") { From 7ac4b6ce031ff2c256e29f3b3d2e903e854a4716 Mon Sep 17 00:00:00 2001 From: oborysevych Date: Fri, 25 Nov 2022 14:03:23 +0200 Subject: [PATCH 095/269] verbose --- playground/terraform/build.gradle.kts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 9e43cb7bead6..10536c1493f7 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -471,6 +471,11 @@ helm { chartName.set("playground") sourceDir.set(file("../infrastructure/helm-playground")) } + + extraArgs { + addAll("-v", "1") + } + releases { create("playground") { from(playground) @@ -493,22 +498,22 @@ task ("gkebackend") { val takeConfig = tasks.getByName("takeConfig") val echo = tasks.getByName("echo") //val back = tasks.getByName("pushBack") - val front = tasks.getByName("pushFront") + //val front = tasks.getByName("pushFront") //val indexcreate = tasks.getByName("indexcreate") - //val helm = tasks.getByName("helmInstallPlayground") + val helm = tasks.getByName("helmInstallPlayground") dependsOn(init) // dependsOn(apply) dependsOn(takeConfig) dependsOn(echo) //dependsOn(back) - dependsOn(front) + //dependsOn(front) //dependsOn(indexcreate) - //dependsOn(helm) + dependsOn(helm) // apply.mustRunAfter(init) takeConfig.mustRunAfter(init) echo.mustRunAfter(takeConfig) //back.mustRunAfter(echo) - front.mustRunAfter(takeConfig) + //front.mustRunAfter(takeConfig) //indexcreate.mustRunAfter(front) - //helm.mustRunAfter(indexcreate) + helm.mustRunAfter(echo) } From f18c830e59c56d37789c59a22f27377864d3d94a Mon Sep 17 00:00:00 2001 From: oborysevych Date: Fri, 25 Nov 2022 14:11:59 +0200 Subject: [PATCH 096/269] remove extra args --- playground/terraform/build.gradle.kts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 10536c1493f7..2fbb6d1d192e 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -472,10 +472,6 @@ helm { sourceDir.set(file("../infrastructure/helm-playground")) } - extraArgs { - addAll("-v", "1") - } - releases { create("playground") { from(playground) From 22ab4bc6219dcc7f919f8b75e56375d67f5e55ba Mon Sep 17 00:00:00 2001 From: oborysevych Date: Fri, 25 Nov 2022 14:12:54 +0200 Subject: [PATCH 097/269] remove commented code --- playground/build.gradle.kts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/playground/build.gradle.kts b/playground/build.gradle.kts index 01ff69ae43c8..b458ef1f7510 100644 --- a/playground/build.gradle.kts +++ b/playground/build.gradle.kts @@ -18,11 +18,6 @@ description = "Apache Beam :: Playground" -// subprojects { -// // Use verbose logging on all Helm commands -// helm.extraArgs.addAll("-v", "1") -// } - // generate protobuf client and server stubs task("generateProto") { group = "build" From a6d8da57c878f7594a0f0658c309b25ffff2978e Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 17:21:00 +0500 Subject: [PATCH 098/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 59 ++++++++++++++------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 9e43cb7bead6..bea5a1c34fd1 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -466,49 +466,50 @@ dns_name: ${dns_name} """) } } -helm { - val playground by charts.creating { - chartName.set("playground") - sourceDir.set(file("../infrastructure/helm-playground")) - } - releases { - create("playground") { - from(playground) - } - } -} -task ("echo") { +//helm { +// val playground by charts.creating { +// chartName.set("playground") +// sourceDir.set(file("../infrastructure/helm-playground")) +// } +// releases { +// create("playground") { +// from(playground) +// } +// } +//} + +task ("helm") { doLast{ exec { - executable("cat") - args("../infrastructure/helm-playground/values.yaml") + executable("helm") + args("install", "../infrastructure/helm-playground/values.yaml") } } } task ("gkebackend") { group = "deploy" val init = tasks.getByName("terraformInit") -// val apply = tasks.getByName("terraformApplyInf") + //val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") - val echo = tasks.getByName("echo") + //val echo = tasks.getByName("echo") //val back = tasks.getByName("pushBack") - val front = tasks.getByName("pushFront") - //val indexcreate = tasks.getByName("indexcreate") - //val helm = tasks.getByName("helmInstallPlayground") + //val front = tasks.getByName("pushFront") + val indexcreate = tasks.getByName("indexcreate") + val helm = tasks.getByName("helm") dependsOn(init) -// dependsOn(apply) + //dependsOn(apply) dependsOn(takeConfig) - dependsOn(echo) + //dependsOn(echo) //dependsOn(back) - dependsOn(front) - //dependsOn(indexcreate) - //dependsOn(helm) -// apply.mustRunAfter(init) + //dependsOn(front) + dependsOn(indexcreate) + dependsOn(helm) + //apply.mustRunAfter(init) takeConfig.mustRunAfter(init) - echo.mustRunAfter(takeConfig) + //echo.mustRunAfter(takeConfig) //back.mustRunAfter(echo) - front.mustRunAfter(takeConfig) - //indexcreate.mustRunAfter(front) - //helm.mustRunAfter(indexcreate) + //front.mustRunAfter(takeConfig) + indexcreate.mustRunAfter(takeConfig) + helm.mustRunAfter(indexcreate) } From 0b952604bab92f85afc9969c904b77ebe4c3ebce Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 17:22:07 +0500 Subject: [PATCH 099/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index bea5a1c34fd1..ae8b74d6b361 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -483,7 +483,7 @@ task ("helm") { doLast{ exec { executable("helm") - args("install", "../infrastructure/helm-playground/values.yaml") + args("install", "../infrastructure/helm-playground") } } } From f704876b0672d43655d97d162cdeef4d51e9ad14 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 17:32:15 +0500 Subject: [PATCH 100/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index ae8b74d6b361..91ec2197696a 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -483,7 +483,7 @@ task ("helm") { doLast{ exec { executable("helm") - args("install", "../infrastructure/helm-playground") + args("install", "--debug", "../infrastructure/helm-playground/Chart.yaml") } } } From bffeaf417da82a28399a84ad4987836c81745e7b Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 17:41:26 +0500 Subject: [PATCH 101/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 91ec2197696a..bae19fcf4906 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -483,7 +483,7 @@ task ("helm") { doLast{ exec { executable("helm") - args("install", "--debug", "../infrastructure/helm-playground/Chart.yaml") + args("install", "playground", "../infrastructure/helm-playground ") } } } From 7f2a6e2ebf4cd737ae315520d147499672759a48 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 17:43:13 +0500 Subject: [PATCH 102/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index bae19fcf4906..979f1779c63b 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -483,7 +483,7 @@ task ("helm") { doLast{ exec { executable("helm") - args("install", "playground", "../infrastructure/helm-playground ") + args("install", "playground", "../infrastructure/helm-playground") } } } From ef4403e6c2a19de801beeed32077c9037e6a83d1 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 17:51:53 +0500 Subject: [PATCH 103/269] Update cloudbuild_pg_to_gke.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index e29330bcf129..58abf0fdbee5 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -37,8 +37,7 @@ steps: printf \ 'bucket = "$_STATE_BUCKET"'\ > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend - terraform version - helm version + gcloud container clusters get-credentials --region 'us-central1' '$_GKE_NAME' --project '$PROJECT_ID' ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" From cd03dd81985dff12122adae9020319b14b4fd64c Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 18:02:09 +0500 Subject: [PATCH 104/269] Update cloudbuild_pg_to_gke.yaml --- playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 58abf0fdbee5..9b48c732bfae 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -37,7 +37,7 @@ steps: printf \ 'bucket = "$_STATE_BUCKET"'\ > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend - gcloud container clusters get-credentials --region 'us-central1' '$_GKE_NAME' --project '$PROJECT_ID' + gcloud container clusters get-credentials --region 'us-central1-a' '$_GKE_NAME' --project '$PROJECT_ID' ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" From ae9cfe277f8b690a578bf2c1e90e3fbf951acf01 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 18:14:32 +0500 Subject: [PATCH 105/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 28 ++++++++++----------------- 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 979f1779c63b..0d5b4e683f81 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -467,26 +467,18 @@ dns_name: ${dns_name} } } -//helm { -// val playground by charts.creating { -// chartName.set("playground") -// sourceDir.set(file("../infrastructure/helm-playground")) -// } -// releases { -// create("playground") { -// from(playground) -// } -// } -//} - -task ("helm") { - doLast{ - exec { - executable("helm") - args("install", "playground", "../infrastructure/helm-playground") +helm { + val playground by charts.creating { + chartName.set("playground") + sourceDir.set(file("../infrastructure/helm-playground")) + } + releases { + create("playground") { + from(playground) } } } + task ("gkebackend") { group = "deploy" val init = tasks.getByName("terraformInit") @@ -496,7 +488,7 @@ task ("gkebackend") { //val back = tasks.getByName("pushBack") //val front = tasks.getByName("pushFront") val indexcreate = tasks.getByName("indexcreate") - val helm = tasks.getByName("helm") + val helm = tasks.getByName("helmInstallPlayground") dependsOn(init) //dependsOn(apply) dependsOn(takeConfig) From 75a55edda73bf91462558415a0766c3ce63a1e42 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 18:23:11 +0500 Subject: [PATCH 106/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 0d5b4e683f81..fef8aab8107e 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -482,26 +482,23 @@ helm { task ("gkebackend") { group = "deploy" val init = tasks.getByName("terraformInit") - //val apply = tasks.getByName("terraformApplyInf") + val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") - //val echo = tasks.getByName("echo") - //val back = tasks.getByName("pushBack") - //val front = tasks.getByName("pushFront") + val back = tasks.getByName("pushBack") + val front = tasks.getByName("pushFront") val indexcreate = tasks.getByName("indexcreate") val helm = tasks.getByName("helmInstallPlayground") dependsOn(init) - //dependsOn(apply) + dependsOn(apply) dependsOn(takeConfig) - //dependsOn(echo) - //dependsOn(back) - //dependsOn(front) + dependsOn(back) + dependsOn(front) dependsOn(indexcreate) dependsOn(helm) - //apply.mustRunAfter(init) - takeConfig.mustRunAfter(init) - //echo.mustRunAfter(takeConfig) - //back.mustRunAfter(echo) - //front.mustRunAfter(takeConfig) - indexcreate.mustRunAfter(takeConfig) + apply.mustRunAfter(init) + takeConfig.mustRunAfter(apply) + back.mustRunAfter(takeConfig) + front.mustRunAfter(back) + indexcreate.mustRunAfter(front) helm.mustRunAfter(indexcreate) } From 686802fe37a1f1a12cbaf2ce3e4129ef65f6b655 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 18:33:04 +0500 Subject: [PATCH 107/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index fef8aab8107e..892f32377d8c 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -482,21 +482,21 @@ helm { task ("gkebackend") { group = "deploy" val init = tasks.getByName("terraformInit") - val apply = tasks.getByName("terraformApplyInf") + //val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") val back = tasks.getByName("pushBack") val front = tasks.getByName("pushFront") val indexcreate = tasks.getByName("indexcreate") val helm = tasks.getByName("helmInstallPlayground") dependsOn(init) - dependsOn(apply) + //dependsOn(apply) dependsOn(takeConfig) dependsOn(back) dependsOn(front) dependsOn(indexcreate) dependsOn(helm) - apply.mustRunAfter(init) - takeConfig.mustRunAfter(apply) + //apply.mustRunAfter(init) + takeConfig.mustRunAfter(init) back.mustRunAfter(takeConfig) front.mustRunAfter(back) indexcreate.mustRunAfter(front) From 7775eb50ea8577bade5ee5d374a921f54ca9c7e3 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 19:23:30 +0500 Subject: [PATCH 108/269] Updates related to PR #24144 --- playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml | 2 +- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index 9b48c732bfae..bd4202180697 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -37,7 +37,7 @@ steps: printf \ 'bucket = "$_STATE_BUCKET"'\ > playground/terraform/environment/$_ENVIRONMENT_NAME/state.tfbackend - gcloud container clusters get-credentials --region 'us-central1-a' '$_GKE_NAME' --project '$PROJECT_ID' + gcloud container clusters get-credentials --region '$_PLAYGROUND_LOCATION' '$_GKE_NAME' --project '$PROJECT_ID' ./gradlew playground:terraform:prepareConfig -Pdns-name="$_DNS_NAME" ./gradlew playground:terraform:gkebackend -Pdocker-repository-root="$_DOCKER_REPOSITORY_ROOT" \ -Pproject_environment="$_ENVIRONMENT_NAME" -Pdocker-tag="$_TAG" -Psdk-tag="$_SDK_TAG" -Pdns-name="$_DNS_NAME" diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index fd660a745c67..e4c9d95b67b5 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -64,7 +64,7 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi GOOGLE_PROJECT="project_id" \ GOOGLE_REGION="us-central1" \ PLAYGROUND_REGION="us-central1" \ - PLAYGROUND_LOCATION="us-central1-b" \ + PLAYGROUND_LOCATION="us-central1-a" \ ENVIRONMENT_NAME="env_name" \ DNS_NAME="dns_name" \ NETWORK_NAME="network_name" \ From 724cee43632479f73d4d1fc845c773aa35428449 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 25 Nov 2022 19:48:40 +0500 Subject: [PATCH 109/269] Updates related to PR #24144 --- .../cloudbuild/cloudbuild_pg_infra.yaml | 1 - .../cloudbuild/cloudbuild_pg_to_gke.yaml | 1 - playground/infrastructure/cloudbuild/env_init.sh | 12 ++++-------- playground/terraform/build.gradle.kts | 2 -- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml index 44371c90a838..e2577b66a00b 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml @@ -32,7 +32,6 @@ steps: 'region = "$_PLAYGROUND_REGION"' \ 'location = "$_PLAYGROUND_LOCATION"' \ 'state_bucket = "$_STATE_BUCKET"' \ - 'bucket_examples_name = "$_STATE_BUCKET-examples"' \ > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars printf \ 'bucket = "$_STATE_BUCKET"'\ diff --git a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml index bd4202180697..42965dffcab4 100644 --- a/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml +++ b/playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml @@ -32,7 +32,6 @@ steps: 'region = "$_PLAYGROUND_REGION"' \ 'location = "$_PLAYGROUND_LOCATION"' \ 'state_bucket = "$_STATE_BUCKET"' \ - 'bucket_examples_name = "$_STATE_BUCKET-examples"' \ > playground/terraform/environment/$_ENVIRONMENT_NAME/terraform.tfvars printf \ 'bucket = "$_STATE_BUCKET"'\ diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index a7179312bd5f..ca167ca97e3f 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -32,14 +32,10 @@ curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts chmod +x get_helm.sh && ./get_helm.sh > /dev/null # Install Terraform -#wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg -#echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ -#| tee /etc/apt/sources.list.d/hashicorp.list -#apt update -y > /dev/null && apt install -y terraform > /dev/null - -curl -OL https://releases.hashicorp.com/terraform/1.0.9/terraform_1.0.9_linux_amd64.zip -unzip terraform_1.0.9_linux_amd64.zip -mv terraform /usr/local/bin +wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ +| tee /etc/apt/sources.list.d/hashicorp.list +apt update -y > /dev/null && apt install -y terraform > /dev/null # Install kubectl curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 892f32377d8c..45b19a069fc1 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -41,9 +41,7 @@ val licenseText = "############################################################# plugins { id("com.pswidersk.terraform-plugin") version "1.0.0" id("org.unbroken-dome.helm") version "1.7.0" - id("org.unbroken-dome.helm-commands") version "1.7.0" id("org.unbroken-dome.helm-releases") version "1.7.0" - id("org.unbroken-dome.helm-publish") version "1.7.0" } terraformPlugin { From 38993af053c8f7265c2680478ec9747601a831f9 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 28 Nov 2022 17:25:35 +0500 Subject: [PATCH 110/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index e4c9d95b67b5..61e8be23c5d9 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -71,7 +71,7 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi GKE_NAME="gke_name" \ TAG="tag_name" \ SDK_TAG="2.43.0" \ - DOCKER_REPOSITORY_ROOT="$_PLAYGROUND_REGION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" + DOCKER_REPOSITORY_ROOT="$PLAYGROUND_REGION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" ``` **Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. From 1c50bb9ff03021d1afbf5a967298a877fedece62 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 28 Nov 2022 18:28:35 +0500 Subject: [PATCH 111/269] Update build.gradle.kts --- playground/terraform/build.gradle.kts | 3 --- 1 file changed, 3 deletions(-) diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts index 45b19a069fc1..b73752ba0f17 100644 --- a/playground/terraform/build.gradle.kts +++ b/playground/terraform/build.gradle.kts @@ -480,20 +480,17 @@ helm { task ("gkebackend") { group = "deploy" val init = tasks.getByName("terraformInit") - //val apply = tasks.getByName("terraformApplyInf") val takeConfig = tasks.getByName("takeConfig") val back = tasks.getByName("pushBack") val front = tasks.getByName("pushFront") val indexcreate = tasks.getByName("indexcreate") val helm = tasks.getByName("helmInstallPlayground") dependsOn(init) - //dependsOn(apply) dependsOn(takeConfig) dependsOn(back) dependsOn(front) dependsOn(indexcreate) dependsOn(helm) - //apply.mustRunAfter(init) takeConfig.mustRunAfter(init) back.mustRunAfter(takeConfig) front.mustRunAfter(back) From 0b63cc43562a232c1f786593d1e79a5307bddfd6 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 28 Nov 2022 21:02:13 +0500 Subject: [PATCH 112/269] Update README.md --- .../cloudbuild-manual-setup/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 61e8be23c5d9..ff7220bc50dc 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -31,7 +31,7 @@ This directory organizes Infrastructure-as-Code to provision dependent resources - Security Admin - Service Account User - [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) -- An existing GCP Bucket to save Terraform state - `state_bucket` +- An existing GCP Bucket to save Terraform state - `state-bucket` - DNS name for your Playground deployment instance - [Terraform](https://www.terraform.io/) - [Apache Beam GitHub](https://github.com/apache/beam) repository cloned locally @@ -60,16 +60,16 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi - `DOCKER_REPOSITORY_ROOT`: GCP Artifact Registry repository name to store Playground container images ```console - export STATE_BUCKET="state_bucket" \ - GOOGLE_PROJECT="project_id" \ + export STATE_BUCKET="state-bucket" \ + GOOGLE_PROJECT="project-id" \ GOOGLE_REGION="us-central1" \ PLAYGROUND_REGION="us-central1" \ PLAYGROUND_LOCATION="us-central1-a" \ - ENVIRONMENT_NAME="env_name" \ - DNS_NAME="dns_name" \ - NETWORK_NAME="network_name" \ - GKE_NAME="gke_name" \ - TAG="tag_name" \ + ENVIRONMENT_NAME="env-name" \ + DNS_NAME="dns-name" \ + NETWORK_NAME="network-name" \ + GKE_NAME="gke-cluster-name" \ + TAG="tag-name" \ SDK_TAG="2.43.0" \ DOCKER_REPOSITORY_ROOT="$PLAYGROUND_REGION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" ``` @@ -147,7 +147,7 @@ terraform apply \ 2. Open Trigger: `Playground-to-gke-trigger`. 3. Scroll down to `Source` - `Repository` to ensure that Apache Beam GitHub repository is connected. - Click on drop-down menu and press `CONNECT NEW REPOSITORY` in case it was not automatically connected. -5. Click `Save` and Run the trigger `Playground-to-gke-trigger`. +4. Click `Save` and Run the trigger `Playground-to-gke-trigger`. ## 6. Validate Playground deployment From 070cd5743960f82df4e9ed5d867b53885ea8a43d Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 13:32:38 +0500 Subject: [PATCH 113/269] Update README.md --- .../infrastructure/cloudbuild-manual-setup/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index ff7220bc50dc..e3ee08310da2 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -70,8 +70,10 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi NETWORK_NAME="network-name" \ GKE_NAME="gke-cluster-name" \ TAG="tag-name" \ - SDK_TAG="2.43.0" \ - DOCKER_REPOSITORY_ROOT="$PLAYGROUND_REGION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" + SDK_TAG="2.43.0" + + export DOCKER_REPOSITORY_ROOT=\ + "$PLAYGROUND_REGION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" ``` **Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. From b4dd9f391dd7000e562b6666296fea6de7193588 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 17:16:26 +0500 Subject: [PATCH 114/269] Update README.md --- .../terraform/infrastructure/cloudbuild-manual-setup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index e3ee08310da2..b41219607eb0 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -71,7 +71,7 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi GKE_NAME="gke-cluster-name" \ TAG="tag-name" \ SDK_TAG="2.43.0" - + export DOCKER_REPOSITORY_ROOT=\ "$PLAYGROUND_REGION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" ``` From afddc1669f64031a39009b7d5808683634e32415 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 22:48:01 +0500 Subject: [PATCH 115/269] Update variables.tf --- playground/terraform/infrastructure/gke/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/gke/variables.tf b/playground/terraform/infrastructure/gke/variables.tf index 97d3b1baa701..e5b75f9530d3 100644 --- a/playground/terraform/infrastructure/gke/variables.tf +++ b/playground/terraform/infrastructure/gke/variables.tf @@ -23,7 +23,7 @@ variable "project_id" { variable "machine_type" { description = "Node pool machine types , for prod set c2d-highcpu-16" - default = "e2-standard-4" + default = "n2-standard-2" } variable "node_count" { From 119d31ba5fe8a78e44ebfa474601a8d6f0bee8ef Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 23:42:44 +0500 Subject: [PATCH 116/269] Update main.tf --- playground/terraform/infrastructure/gke/main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playground/terraform/infrastructure/gke/main.tf b/playground/terraform/infrastructure/gke/main.tf index bdc11d1b6b29..112cea5af52e 100644 --- a/playground/terraform/infrastructure/gke/main.tf +++ b/playground/terraform/infrastructure/gke/main.tf @@ -24,6 +24,9 @@ resource "google_container_cluster" "playground-gke" { initial_node_count = var.node_count network = var.network subnetwork = var.subnetwork + + enable_autopilot = true + node_config { machine_type = var.machine_type service_account = var.service_account_email From 8ec1c0bca5a5fe4a0ac38fe2be07b278ab2c700f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 23:57:55 +0500 Subject: [PATCH 117/269] Revert "Update main.tf" This reverts commit 119d31ba5fe8a78e44ebfa474601a8d6f0bee8ef. --- playground/terraform/infrastructure/gke/main.tf | 3 --- 1 file changed, 3 deletions(-) diff --git a/playground/terraform/infrastructure/gke/main.tf b/playground/terraform/infrastructure/gke/main.tf index 112cea5af52e..bdc11d1b6b29 100644 --- a/playground/terraform/infrastructure/gke/main.tf +++ b/playground/terraform/infrastructure/gke/main.tf @@ -24,9 +24,6 @@ resource "google_container_cluster" "playground-gke" { initial_node_count = var.node_count network = var.network subnetwork = var.subnetwork - - enable_autopilot = true - node_config { machine_type = var.machine_type service_account = var.service_account_email From 4fe36a506d3937f3ecb3110303d65690c4aa4af0 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 23:58:01 +0500 Subject: [PATCH 118/269] Revert "Update variables.tf" This reverts commit afddc1669f64031a39009b7d5808683634e32415. --- playground/terraform/infrastructure/gke/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/terraform/infrastructure/gke/variables.tf b/playground/terraform/infrastructure/gke/variables.tf index e5b75f9530d3..97d3b1baa701 100644 --- a/playground/terraform/infrastructure/gke/variables.tf +++ b/playground/terraform/infrastructure/gke/variables.tf @@ -23,7 +23,7 @@ variable "project_id" { variable "machine_type" { description = "Node pool machine types , for prod set c2d-highcpu-16" - default = "n2-standard-2" + default = "e2-standard-4" } variable "node_count" { From a62b05d1bb479a73f515acb9049dd25bc92b7d98 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 30 Nov 2022 17:07:39 +0500 Subject: [PATCH 119/269] Majority of updates following the comments in #24144 --- playground/infrastructure/cloudbuild/env_init.sh | 8 ++------ .../cloudbuild-manual-setup/02.builders/triggers.tf | 4 ++-- .../cloudbuild-manual-setup/02.builders/variables.tf | 10 +++++----- .../infrastructure/cloudbuild-manual-setup/README.md | 10 +++++++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/playground/infrastructure/cloudbuild/env_init.sh b/playground/infrastructure/cloudbuild/env_init.sh index ca167ca97e3f..c612472433bd 100644 --- a/playground/infrastructure/cloudbuild/env_init.sh +++ b/playground/infrastructure/cloudbuild/env_init.sh @@ -15,11 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -GO_VERSION=1.18 - apt-get update > /dev/null -# Install JRE +# Install dependencies apt-get install -y build-essential unzip apt-transport-https ca-certificates curl software-properties-common gnupg2 wget > /dev/null # Install Docker @@ -42,7 +40,5 @@ curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s ht && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl -# Install golang & jdk -curl -O https://dl.google.com/go/go"$GO_VERSION".linux-amd64.tar.gz -tar -C /usr/local -xvf go"$GO_VERSION".linux-amd64.tar.gz > /dev/null +# Install jdk apt-get install openjdk-11-jdk -y > /dev/null \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf index 343e168dc383..aea6d28f8547 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf @@ -39,7 +39,7 @@ resource "google_cloudbuild_trigger" "playground_infrastructure" { substitutions = { _PLAYGROUND_REGION : var.playground_region - _PLAYGROUND_LOCATION : var.playground_location + _PLAYGROUND_LOCATION : var.playground_zone _ENVIRONMENT_NAME : var.playground_environment_name _DNS_NAME : var.playground_dns_name _NETWORK_NAME : var.playground_network_name @@ -70,7 +70,7 @@ resource "google_cloudbuild_trigger" "playground_to_gke" { substitutions = { _PLAYGROUND_REGION : var.playground_region - _PLAYGROUND_LOCATION : var.playground_location + _PLAYGROUND_LOCATION : var.playground_zone _ENVIRONMENT_NAME : var.playground_environment_name _DNS_NAME : var.playground_dns_name _NETWORK_NAME : var.playground_network_name diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf index 56ee9d3a84a7..f03d1d4599f8 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf @@ -28,13 +28,13 @@ variable "region" { variable "infra_trigger_name" { type = string description = "The name of the trigger that will deploy Playground infrastructure" - default = "Playground-infrastructure-trigger" + default = "playground-infrastructure-trigger" } variable "gke_trigger_name" { type = string description = "The name of the trigger that will deploy Playground to GKE" - default = "Playground-to-gke-trigger" + default = "playground-to-gke-trigger" } variable "cloudbuild_service_account_id" { @@ -72,11 +72,11 @@ variable "docker_repository_root" { } variable "playground_region" { - description = "Region (us-central1) where playground infrastructure will be deployed" + description = "Region (For example: us-central1) where playground infrastructure will be deployed" } -variable "playground_location" { - description = "Location (us-central1-b) where playground infrastructure will be deployed" +variable "playground_zone" { + description = "Zone (For example: us-central1-b) where playground infrastructure will be deployed" } variable "sdk_tag" { diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index b41219607eb0..52c80c4d3bad 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -19,11 +19,15 @@ This directory organizes Infrastructure-as-Code to provision dependent resources and set up Cloud Build for Beam Playground. +Playground deployment main data and scripts can be found by next locations: +- [Playground deployment main directory](https://github.com/apache/beam/blob/master/playground/) +- [Playground deployment main README file](https://github.com/apache/beam/blob/master/playground/terraform/README.md) + ## Requirements: - [GCP project](https://cloud.google.com/resource-manager/docs/creating-managing-projects) - [GCP User account](https://cloud.google.com/appengine/docs/standard/access-control?tab=python) _(Note: You will find the instruction "How to create User account" for your new project)_
- Ensure that the account has at least the following privileges: + Ensure that the account has at least the following [IAM roles](https://cloud.google.com/iam/docs/understanding-roles): - Service Account Admin - Storage Admin - Service Usage Admin @@ -92,7 +96,7 @@ cd playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/ # Run terraform scripts terraform init -backend-config="bucket=$STATE_BUCKET" -terraform apply -var "project_id=$GOOGLE_PROJECT" +terraform apply -var="project_id=$(gcloud config get-value project)" ``` ## 2. Connect Apache Beam GitHub repository and GCP Cloud Build @@ -117,7 +121,7 @@ cd ../02.builders terraform init -backend-config="bucket=$STATE_BUCKET" terraform apply \ --var "project_id=$GOOGLE_PROJECT" \ +-var "project_id=$(gcloud config get-value project)" \ -var "region=$GOOGLE_REGION" \ -var "playground_region=$PLAYGROUND_REGION" \ -var "playground_location=$PLAYGROUND_LOCATION" \ From bc004b241f7bca6f44910682732be4029a3fb80a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 30 Nov 2022 17:39:31 +0500 Subject: [PATCH 120/269] Rest of updates following the comments in #24144 --- .../02.builders/triggers.tf | 34 +++++------ .../02.builders/variables.tf | 33 ++++++++--- .../cloudbuild-manual-setup/README.md | 59 +++---------------- 3 files changed, 49 insertions(+), 77 deletions(-) diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf index aea6d28f8547..335cfe12edec 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf @@ -26,15 +26,12 @@ resource "google_cloudbuild_trigger" "playground_infrastructure" { description = "Builds the base image and then runs cloud build config file to deploy Playground infrastructure" - source_to_build { - uri = "https://github.com/apache/beam" - ref = "refs/heads/master" - repo_type = "GITHUB" - } - - git_file_source { - path = "playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml" - repo_type = "GITHUB" + github { + owner = var.github_repository_owner + name = var.github_repository_name + push { + branch = var.github_repository_branch + } } substitutions = { @@ -47,6 +44,8 @@ resource "google_cloudbuild_trigger" "playground_infrastructure" { _STATE_BUCKET : var.state_bucket } + filename = "playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml" + service_account = data.google_service_account.cloudbuild_sa.id } @@ -57,15 +56,12 @@ resource "google_cloudbuild_trigger" "playground_to_gke" { description = "Builds the base image and then runs cloud build config file to deploy Playground to GKE" - source_to_build { - uri = "https://github.com/apache/beam" - ref = "refs/heads/master" - repo_type = "GITHUB" - } - - git_file_source { - path = "playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml" - repo_type = "GITHUB" + github { + owner = var.github_repository_owner + name = var.github_repository_name + push { + branch = var.github_repository_branch + } } substitutions = { @@ -81,5 +77,7 @@ resource "google_cloudbuild_trigger" "playground_to_gke" { _SDK_TAG : var.sdk_tag } + filename = "playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml" + service_account = data.google_service_account.cloudbuild_sa.id } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf index f03d1d4599f8..b8462217422b 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf @@ -43,24 +43,39 @@ variable "cloudbuild_service_account_id" { default = "playground-cloudbuild-sa" } +variable "github_repository_name" { + type = string + description = "The name of the GitHub repository. For example the repository name for https://github.com/example/foo is 'foo'." +} + +variable "github_repository_owner" { + type = string + description = "Owner of the GitHub repository. For example the owner for https://github.com/example/foo is 'example'." +} + +variable "github_repository_branch" { + type = string + description = "The GitHub repository branch regex to match cloud build trigger" +} + variable "playground_environment_name" { - description = "Environment where to deploy Playground. Located in playground/terraform/environment/{environment_name}" + description = "Environment where to deploy Playground. Located in playground/terraform/environment/{environment_name}. E.g. test, dev, prod" } variable "playground_dns_name" { - description = "DNS record name for Playground website" + description = "DNS record name for Playground website. More details: https://github.com/apache/beam/blob/master/playground/terraform/README.md#deploy-playground-infrastructure" } variable "playground_network_name" { - description = "GCP VPC Network Name for Playground deployment" + description = "The Google Cloud Platform (GCP) VPC Network Name for Playground deployment" } variable "playground_gke_name" { - description = "Playground GKE Cluster name" + description = "Playground GKE Cluster name in Google Cloud Platform (GCP)" } variable "state_bucket" { - description = "GCS bucket name for Beam Playground temp files and Terraform state" + description = "The Google Cloud Platform (GCP) GCS bucket name for Beam Playground temp files and Terraform state" } variable "image_tag" { @@ -68,17 +83,17 @@ variable "image_tag" { } variable "docker_repository_root" { - description = "The name of GCP Artifact Registry Repository where Playground images will be saved to" + description = "The name of Google Cloud Platform (GCP) Artifact Registry Repository where Playground images will be saved to" } variable "playground_region" { - description = "Region (For example: us-central1) where playground infrastructure will be deployed" + description = "The Google Cloud Platform (GCP) region (For example: us-central1) where playground infrastructure will be deployed to" } variable "playground_zone" { - description = "Zone (For example: us-central1-b) where playground infrastructure will be deployed" + description = "The Google Cloud Platform (GCP) zone (For example: us-central1-b) where playground infrastructure will be deployed to" } variable "sdk_tag" { - description = "Apache Beam go python image sdk tag" + description = "Apache Beam Golang and Python images SDK tag. See more: https://hub.docker.com/r/apache/beam_python3.7_sdk/tags and https://hub.docker.com/r/apache/beam_go_sdk" } \ No newline at end of file diff --git a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md index 52c80c4d3bad..fdb2704460e2 100644 --- a/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md +++ b/playground/terraform/infrastructure/cloudbuild-manual-setup/README.md @@ -49,52 +49,25 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup` provi #### To execute the module: -1. Set following environment variables: - - `STATE_BUCKET`: GCP Storage bucket name to save Terraform state - - `GOOGLE_PROJECT`: GCP Project ID - - `GOOGLE_REGION`: GCP region to save triggers to - - `PLAYGROUND_REGION`: GCP Region (us-central1) where playground infrastructure will be deployed - - `PLAYGROUND_LOCATION`: GCP Location (us-central1-b) where playground infrastructure will be deployed - - `ENVIRONMENT_NAME`: Environment where Playground will be deployed (located at playground/terraform/environment/environment_name) - - `DNS_NAME`: DNS for deployed Playground webpage - - `NETWORK_NAME`: GCP VPC Network Name for Playground deployment - - `GKE_NAME`: Playground GKE Cluster name - - `TAG`: Tag for Playground images - - `SDK_TAG`: Tag for SDK images of Apache Beam Go and Python (typically last stable 2.42.0, 2.43.0) - - `DOCKER_REPOSITORY_ROOT`: GCP Artifact Registry repository name to store Playground container images - -```console - export STATE_BUCKET="state-bucket" \ - GOOGLE_PROJECT="project-id" \ - GOOGLE_REGION="us-central1" \ - PLAYGROUND_REGION="us-central1" \ - PLAYGROUND_LOCATION="us-central1-a" \ - ENVIRONMENT_NAME="env-name" \ - DNS_NAME="dns-name" \ - NETWORK_NAME="network-name" \ - GKE_NAME="gke-cluster-name" \ - TAG="tag-name" \ - SDK_TAG="2.43.0" - - export DOCKER_REPOSITORY_ROOT=\ - "$PLAYGROUND_REGION-docker.pkg.dev/$GOOGLE_PROJECT/playground-repository" -``` **Note:** Please see [Cloud Build locations](https://cloud.google.com/build/docs/locations) for the list of all supported locations. -2. Run commands: +1. Run commands: ```console +# Set environment variable for state bucket +export STATE_BUCKET="state-bucket" + # Create a new authentication configuration for GCP Project with the created user account gcloud init # Command imports new user account credentials into Application Default Credentials gcloud auth application-default login -# Navigate to 01.setup folder +# Navigate to 01.setup directory cd playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/ -# Run terraform scripts +# Run terraform commands terraform init -backend-config="bucket=$STATE_BUCKET" terraform apply -var="project_id=$(gcloud config get-value project)" ``` @@ -112,27 +85,13 @@ The `playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders` pr #### To execute the module - ``` -# Navigate to playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders folder +# Navigate to playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders directory cd ../02.builders -# Run terraform scripts +# Run terraform commands terraform init -backend-config="bucket=$STATE_BUCKET" - -terraform apply \ --var "project_id=$(gcloud config get-value project)" \ --var "region=$GOOGLE_REGION" \ --var "playground_region=$PLAYGROUND_REGION" \ --var "playground_location=$PLAYGROUND_LOCATION" \ --var "playground_environment_name=$ENVIRONMENT_NAME" \ --var "playground_dns_name=$DNS_NAME" \ --var "playground_network_name=$NETWORK_NAME" \ --var "playground_gke_name=$GKE_NAME" \ --var "state_bucket=$STATE_BUCKET" \ --var "image_tag=$TAG" \ --var "sdk_tag=$SDK_TAG" \ --var "docker_repository_root=$DOCKER_REPOSITORY_ROOT" +terraform apply -var="project_id=$(gcloud config get-value project)" ``` ## 4. Run Cloud Build `Playground-infrastructure-trigger` to deploy Playground infrastructure From 44b409b64ef9f93aa1a80cc74d0e82ab7cbfdd25 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Fri, 7 Oct 2022 06:36:29 -0700 Subject: [PATCH 121/269] Create main.tf --- learning/tour-of-beam/terraform/main.tf | 1 + 1 file changed, 1 insertion(+) create mode 100644 learning/tour-of-beam/terraform/main.tf diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf new file mode 100644 index 000000000000..8b137891791f --- /dev/null +++ b/learning/tour-of-beam/terraform/main.tf @@ -0,0 +1 @@ + From c0e73c7a6a3e1b9abfcfda1c2a4f0a3d1190ba68 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Fri, 7 Oct 2022 06:37:09 -0700 Subject: [PATCH 122/269] Update main.tf --- learning/tour-of-beam/terraform/main.tf | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf index 8b137891791f..3698bec4899b 100644 --- a/learning/tour-of-beam/terraform/main.tf +++ b/learning/tour-of-beam/terraform/main.tf @@ -1 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +resource "google_cloudfunctions_function" "cloud_function" { + name = var.cloud_function_name + runtime = var.runtime_name + region = var.region + service_account_email = google_service_account.sa_cloud_function.email + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + source_archive_bucket = google_storage_bucket.function_bucket.name + source_archive_object = google_storage_bucket_object.zip.name + + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered (def hello_gcs) + entry_point = var.code_function_name + +} From 759f00cf42fd13d14621dbc147effbedb7499c1f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Fri, 7 Oct 2022 06:37:31 -0700 Subject: [PATCH 123/269] Create requirements.txt --- learning/tour-of-beam/terraform/src/requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 learning/tour-of-beam/terraform/src/requirements.txt diff --git a/learning/tour-of-beam/terraform/src/requirements.txt b/learning/tour-of-beam/terraform/src/requirements.txt new file mode 100644 index 000000000000..8b137891791f --- /dev/null +++ b/learning/tour-of-beam/terraform/src/requirements.txt @@ -0,0 +1 @@ + From 8d019f0e1ea250af9bea505b513e1129461036ef Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Fri, 7 Oct 2022 06:38:00 -0700 Subject: [PATCH 124/269] Add files via upload --- learning/tour-of-beam/terraform/iam.tf | 23 +++++++++++ learning/tour-of-beam/terraform/outputs.tf | 22 ++++++++++ learning/tour-of-beam/terraform/storage.tf | 42 ++++++++++++++++++++ learning/tour-of-beam/terraform/variables.tf | 42 ++++++++++++++++++++ 4 files changed, 129 insertions(+) create mode 100644 learning/tour-of-beam/terraform/iam.tf create mode 100644 learning/tour-of-beam/terraform/outputs.tf create mode 100644 learning/tour-of-beam/terraform/storage.tf create mode 100644 learning/tour-of-beam/terraform/variables.tf diff --git a/learning/tour-of-beam/terraform/iam.tf b/learning/tour-of-beam/terraform/iam.tf new file mode 100644 index 000000000000..c16a5f009d81 --- /dev/null +++ b/learning/tour-of-beam/terraform/iam.tf @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +resource "google_service_account" "sa_cloud_function" { + account_id = var.service_account_name + display_name = "Service Account to run Cloud Functions" + project = var.project_id +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/outputs.tf b/learning/tour-of-beam/terraform/outputs.tf new file mode 100644 index 000000000000..7c05c5f32cca --- /dev/null +++ b/learning/tour-of-beam/terraform/outputs.tf @@ -0,0 +1,22 @@ +# +# Licensed to the Atpache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +output "cloud_function_trigger_url" { + value = google_cloudfunctions_function.cloud_function.https_trigger_url +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/storage.tf b/learning/tour-of-beam/terraform/storage.tf new file mode 100644 index 000000000000..d0752aeb6640 --- /dev/null +++ b/learning/tour-of-beam/terraform/storage.tf @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +#Generates archive of source code +data "archive_file" "source" { + type = "zip" + source_dir = "src" + output_path = "/tmp/function.zip" +} + +#Declaring Google Cloud Storage bucket to store the code of the Cloud Function +resource "google_storage_bucket" "function_bucket" { + name = "${var.project_id}-function-bucket" + location = var.region +} + +# Add source code zip to the Cloud Function's bucket +resource "google_storage_bucket_object" "zip" { + # Use an MD5 here. If there's no changes to the source code, this won't change either. + # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing + # a redeployment when it has! + name = "${data.archive_file.source.output_md5}.zip" + bucket = google_storage_bucket.function_bucket.name + source = data.archive_file.source.output_path + content_type = "application/zip" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf new file mode 100644 index 000000000000..926fe3646567 --- /dev/null +++ b/learning/tour-of-beam/terraform/variables.tf @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +variable "project_id" { + description = "The GCP Project ID where ToB Cloud Functions will be created" +} + +variable "region" { + description = "Location of App" +} + +variable "cloud_function_name" { + description = "Name of cloud function" +} + +variable "runtime_name" { + description = "Name of runtime (e.g. nodejs16, python39, dotnet3, go116, java11, ruby30, php74) see https://cloud.google.com/functions/docs/concepts/execution-environment#runtimes for more runtimes" +} + +variable "code_function_name" { + description = "Name of function in code" +} + +variable "service_account_name" { + description = "Name of the service account to run Cloud Function" +} \ No newline at end of file From 46e5ed490874b415569b719b355411eb21de86dc Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Fri, 7 Oct 2022 06:38:39 -0700 Subject: [PATCH 125/269] Added main.py to src/ --- learning/tour-of-beam/terraform/src/main.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 learning/tour-of-beam/terraform/src/main.py diff --git a/learning/tour-of-beam/terraform/src/main.py b/learning/tour-of-beam/terraform/src/main.py new file mode 100644 index 000000000000..9cbde42203f5 --- /dev/null +++ b/learning/tour-of-beam/terraform/src/main.py @@ -0,0 +1,19 @@ +import os +import logging + +def http_handler(request): + """ + HTTP Cloud Function that returns the value of FOO. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + + logging.info('Handler Started!') + + response_body = f'This is a message for Cloud Function HTTP Trigger' + return response_body \ No newline at end of file From fad664b068e7278cfdcd6589352629adc6458b14 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Fri, 7 Oct 2022 06:49:09 -0700 Subject: [PATCH 126/269] Create README.md --- learning/tour-of-beam/terraform/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 learning/tour-of-beam/terraform/README.md diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md new file mode 100644 index 000000000000..ed05bde415b3 --- /dev/null +++ b/learning/tour-of-beam/terraform/README.md @@ -0,0 +1,19 @@ +# GCP Cloud Functions back-end deployment using Terraform + +This README will briefly guide you through the steps of how to use the script. It is pretty easy. + +### Step 1: Create project in GCP + +### Step 2: Create service-account that Cloud Functions will use + +### Step 3: Assign necessary roles and permissions to that account + +### Step 4: Export key of service account as json file + +### Step 5: set ENV GOOGLE_APPLICATION_CREDENTIALS = 'your path to json file' + +### Step 6: Clone/download scripts + +### Step 7: Open terminal, go to terraform scripts directory + +### Step 8: Run the script and assign variables as you need From 8facebbbe3abdb593ab409b5aa0e808aefecd056 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Fri, 7 Oct 2022 06:51:03 -0700 Subject: [PATCH 127/269] Update README.md --- learning/tour-of-beam/terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index ed05bde415b3..da1bd67985c6 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -1,4 +1,4 @@ -# GCP Cloud Functions back-end deployment using Terraform +# (DRAFT) GCP Cloud Functions back-end deployment using Terraform This README will briefly guide you through the steps of how to use the script. It is pretty easy. From 5cbeb80a91f3c129f69e401930278cd26861a168 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 10 Oct 2022 12:21:46 -0700 Subject: [PATCH 128/269] Update tf backend script --- learning/tour-of-beam/terraform/src/main.py | 36 ++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/learning/tour-of-beam/terraform/src/main.py b/learning/tour-of-beam/terraform/src/main.py index 9cbde42203f5..5c3c33f5dd9c 100644 --- a/learning/tour-of-beam/terraform/src/main.py +++ b/learning/tour-of-beam/terraform/src/main.py @@ -1,19 +1,19 @@ -import os -import logging - -def http_handler(request): - """ - HTTP Cloud Function that returns the value of FOO. - Args: - request (flask.Request): The request object. - - Returns: - The response text, or any set of values that can be turned into a - Response object using `make_response` - . - """ - - logging.info('Handler Started!') - - response_body = f'This is a message for Cloud Function HTTP Trigger' +import os +import logging + +def http_handler(request): + """ + HTTP Cloud Function that returns the value of FOO. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + + logging.info('Handler Started!') + + response_body = f'This is a message for Cloud Function HTTP Trigger' return response_body \ No newline at end of file From a53b94504a6569e5b002c691869c895cdaf806ba Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 10 Oct 2022 12:21:46 -0700 Subject: [PATCH 129/269] Update tf backend script --- learning/tour-of-beam/terraform/iam.tf | 10 ++--- learning/tour-of-beam/terraform/main.tf | 46 +++++++++++++++++--- learning/tour-of-beam/terraform/outputs.tf | 12 ++++- learning/tour-of-beam/terraform/storage.tf | 2 +- learning/tour-of-beam/terraform/variables.tf | 20 ++++----- 5 files changed, 67 insertions(+), 23 deletions(-) diff --git a/learning/tour-of-beam/terraform/iam.tf b/learning/tour-of-beam/terraform/iam.tf index c16a5f009d81..1236ab5c8619 100644 --- a/learning/tour-of-beam/terraform/iam.tf +++ b/learning/tour-of-beam/terraform/iam.tf @@ -16,8 +16,8 @@ # specific language governing permissions and limitations # under the License. -resource "google_service_account" "sa_cloud_function" { - account_id = var.service_account_name - display_name = "Service Account to run Cloud Functions" - project = var.project_id -} \ No newline at end of file +#resource "google_service_account" "sa_cloud_function" { +# account_id = var.service_account_name +# display_name = "Service Account to run Cloud Functions" +# project = var.project_id +#} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf index 3698bec4899b..68c8898ec7a3 100644 --- a/learning/tour-of-beam/terraform/main.tf +++ b/learning/tour-of-beam/terraform/main.tf @@ -14,11 +14,16 @@ # limitations under the License. # -resource "google_cloudfunctions_function" "cloud_function" { - name = var.cloud_function_name - runtime = var.runtime_name +provider "google" { + project = var.project_id + region = var.region +} + +resource "google_cloudfunctions_function" "backend_function_1" { + name = "getSdkList-function" + runtime = "go116" region = var.region - service_account_email = google_service_account.sa_cloud_function.email + service_account_email = var.service_account_email ingress_settings = "ALLOW_ALL" # Get the source code of the cloud function as a Zip compression source_archive_bucket = google_storage_bucket.function_bucket.name @@ -26,7 +31,38 @@ resource "google_cloudfunctions_function" "cloud_function" { trigger_http = true # Name of the function that will be executed when the Google Cloud Function is triggered (def hello_gcs) - entry_point = var.code_function_name + entry_point = "getSdkList" } +resource "google_cloudfunctions_function" "backend_function_2" { + name = "getContentTree-function" + runtime = "go116" + region = var.region + service_account_email = var.service_account_email + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + source_archive_bucket = google_storage_bucket.function_bucket.name + source_archive_object = google_storage_bucket_object.zip.name + + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered (def hello_gcs) + entry_point = "getContentTree" + +} + +resource "google_cloudfunctions_function" "backend_function_3" { + name = "getUnitContent-function" + runtime = "go116" + region = var.region + service_account_email = var.service_account_email + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + source_archive_bucket = google_storage_bucket.function_bucket.name + source_archive_object = google_storage_bucket_object.zip.name + + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered (def hello_gcs) + entry_point = "getUnitContent" + +} diff --git a/learning/tour-of-beam/terraform/outputs.tf b/learning/tour-of-beam/terraform/outputs.tf index 7c05c5f32cca..9800ed8c7e56 100644 --- a/learning/tour-of-beam/terraform/outputs.tf +++ b/learning/tour-of-beam/terraform/outputs.tf @@ -17,6 +17,14 @@ # under the License. # -output "cloud_function_trigger_url" { - value = google_cloudfunctions_function.cloud_function.https_trigger_url +output "cloud_function_trigger_url_1" { + value = google_cloudfunctions_function.backend_function_1.https_trigger_url +} + +output "cloud_function_trigger_url_2" { + value = google_cloudfunctions_function.backend_function_2.https_trigger_url +} + +output "cloud_function_trigger_url_3" { + value = google_cloudfunctions_function.backend_function_3.https_trigger_url } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/storage.tf b/learning/tour-of-beam/terraform/storage.tf index d0752aeb6640..1afa0ab27bef 100644 --- a/learning/tour-of-beam/terraform/storage.tf +++ b/learning/tour-of-beam/terraform/storage.tf @@ -20,7 +20,7 @@ #Generates archive of source code data "archive_file" "source" { type = "zip" - source_dir = "src" + source_dir = "../backend" output_path = "/tmp/function.zip" } diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf index 926fe3646567..05ece0042316 100644 --- a/learning/tour-of-beam/terraform/variables.tf +++ b/learning/tour-of-beam/terraform/variables.tf @@ -25,18 +25,18 @@ variable "region" { description = "Location of App" } -variable "cloud_function_name" { - description = "Name of cloud function" -} +#variable "cloud_function_name" { +# description = "Name of cloud function" +#} -variable "runtime_name" { - description = "Name of runtime (e.g. nodejs16, python39, dotnet3, go116, java11, ruby30, php74) see https://cloud.google.com/functions/docs/concepts/execution-environment#runtimes for more runtimes" -} +#variable "runtime_name" { +# description = "Name of runtime (e.g. nodejs16, python39, dotnet3, go116, java11, ruby30, php74) see https://cloud.google.com/functions/docs/concepts/execution-environment#runtimes for more runtimes" +#} -variable "code_function_name" { - description = "Name of function in code" -} +#variable "code_function_name" { +# description = "Name of function in code" +#} -variable "service_account_name" { +variable "service_account_email" { description = "Name of the service account to run Cloud Function" } \ No newline at end of file From d5e30b6c4e4ba4719d620f422ecc90864c7a6b39 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 12 Oct 2022 00:53:29 -0700 Subject: [PATCH 130/269] Update TF scripts --- learning/tour-of-beam/terraform/README.md | 19 ----- .../tour-of-beam/terraform/buckets/main.tf | 29 +++++++ .../tour-of-beam/terraform/buckets/output.tf | 15 ++++ .../terraform/buckets/variables.tf | 18 ++++ .../terraform/cloud-functions/main.tf | 23 ++++++ .../terraform/cloud-functions/output.tf | 11 +++ .../terraform/cloud-functions/variables.tf | 7 ++ learning/tour-of-beam/terraform/iam.tf | 23 ------ learning/tour-of-beam/terraform/main.tf | 82 +++++-------------- learning/tour-of-beam/terraform/output.tf | 7 ++ learning/tour-of-beam/terraform/outputs.tf | 30 ------- learning/tour-of-beam/terraform/provider.tf | 19 +++++ learning/tour-of-beam/terraform/setup/iam.tf | 27 ++++++ .../tour-of-beam/terraform/setup/output.tf | 3 + .../tour-of-beam/terraform/setup/services.tf | 23 ++++++ .../tour-of-beam/terraform/setup/variables.tf | 9 ++ learning/tour-of-beam/terraform/src/main.py | 19 ----- .../terraform/src/requirements.txt | 1 - learning/tour-of-beam/terraform/storage.tf | 42 ---------- learning/tour-of-beam/terraform/variables.tf | 51 +++++------- 20 files changed, 231 insertions(+), 227 deletions(-) create mode 100644 learning/tour-of-beam/terraform/buckets/main.tf create mode 100644 learning/tour-of-beam/terraform/buckets/output.tf create mode 100644 learning/tour-of-beam/terraform/buckets/variables.tf create mode 100644 learning/tour-of-beam/terraform/cloud-functions/main.tf create mode 100644 learning/tour-of-beam/terraform/cloud-functions/output.tf create mode 100644 learning/tour-of-beam/terraform/cloud-functions/variables.tf delete mode 100644 learning/tour-of-beam/terraform/iam.tf create mode 100644 learning/tour-of-beam/terraform/output.tf delete mode 100644 learning/tour-of-beam/terraform/outputs.tf create mode 100644 learning/tour-of-beam/terraform/provider.tf create mode 100644 learning/tour-of-beam/terraform/setup/iam.tf create mode 100644 learning/tour-of-beam/terraform/setup/output.tf create mode 100644 learning/tour-of-beam/terraform/setup/services.tf create mode 100644 learning/tour-of-beam/terraform/setup/variables.tf delete mode 100644 learning/tour-of-beam/terraform/src/main.py delete mode 100644 learning/tour-of-beam/terraform/src/requirements.txt delete mode 100644 learning/tour-of-beam/terraform/storage.tf diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index da1bd67985c6..e69de29bb2d1 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -1,19 +0,0 @@ -# (DRAFT) GCP Cloud Functions back-end deployment using Terraform - -This README will briefly guide you through the steps of how to use the script. It is pretty easy. - -### Step 1: Create project in GCP - -### Step 2: Create service-account that Cloud Functions will use - -### Step 3: Assign necessary roles and permissions to that account - -### Step 4: Export key of service account as json file - -### Step 5: set ENV GOOGLE_APPLICATION_CREDENTIALS = 'your path to json file' - -### Step 6: Clone/download scripts - -### Step 7: Open terminal, go to terraform scripts directory - -### Step 8: Run the script and assign variables as you need diff --git a/learning/tour-of-beam/terraform/buckets/main.tf b/learning/tour-of-beam/terraform/buckets/main.tf new file mode 100644 index 000000000000..b663e73177da --- /dev/null +++ b/learning/tour-of-beam/terraform/buckets/main.tf @@ -0,0 +1,29 @@ +#Generates archive of source code +data "archive_file" "source" { + type = "zip" + source_dir = "../../backend" + output_path = "/tmp/backend.zip" +} + +resource "google_storage_bucket" "function_bucket" { + name = var.name + location = var.location + project = var.project_id + storage_class = var.storage_class +} + +resource "google_storage_bucket_object" "zip" { + # Use an MD5 here. If there's no changes to the source code, this won't change either. + # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing + # a redeployment when it has! + name = "${data.archive_file.source.output_md5}.zip" + bucket = google_storage_bucket.function_bucket.name + source = data.archive_file.source.output_path + content_type = "application/zip" +} + +resource "google_storage_bucket_access_control" "public_rule" { + bucket = google_storage_bucket.function_bucket.name + role = "READER" + entity = "allUsers" +} diff --git a/learning/tour-of-beam/terraform/buckets/output.tf b/learning/tour-of-beam/terraform/buckets/output.tf new file mode 100644 index 000000000000..69bfc4f5c6fb --- /dev/null +++ b/learning/tour-of-beam/terraform/buckets/output.tf @@ -0,0 +1,15 @@ +output "function-bucket-id" { + value = google_storage_bucket.function_bucket.id +} + +output "function-bucket-name" { + value = google_storage_bucket.function_bucket.name +} + +output "function-bucket-object" { + value = google_storage_bucket_object.zip.name +} + +output "function-bucket-location" { + value = google_storage_bucket.function_bucket.location +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/buckets/variables.tf b/learning/tour-of-beam/terraform/buckets/variables.tf new file mode 100644 index 000000000000..647c09f40620 --- /dev/null +++ b/learning/tour-of-beam/terraform/buckets/variables.tf @@ -0,0 +1,18 @@ +variable "name" { + description = "Name of Bucket to Store Cloud Functions" + default = "tour-of-beam-cloudfunction-bucket" +} + +variable "location" { + description = "Cloud Functions Bucket Region" + default = "US" +} + +variable "project_id" { + description = "The GCP Project ID where ToB Cloud Functions will be created" +} + +variable "storage_class" { + description = "Functions Bucket Storage Class" + default = "STANDARD" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/cloud-functions/main.tf b/learning/tour-of-beam/terraform/cloud-functions/main.tf new file mode 100644 index 000000000000..a926ef71c19e --- /dev/null +++ b/learning/tour-of-beam/terraform/cloud-functions/main.tf @@ -0,0 +1,23 @@ +module "setup" { + source = "../setup" +} + +module "buckets" { + source = "../buckets" +} + +resource "google_cloudfunctions_function" "cloud_function" { + name = "tour-of-beam-backend-cloud-function" + runtime = "go116" + region = var.region + service_account_email = module.setup.service-account-email + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + source_archive_bucket = module.buckets.function-bucket-name + source_archive_object = module.buckets.function-bucket-object + + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered (def hello_gcs) + entry_point = "init" + +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/cloud-functions/output.tf b/learning/tour-of-beam/terraform/cloud-functions/output.tf new file mode 100644 index 000000000000..1d42a873e283 --- /dev/null +++ b/learning/tour-of-beam/terraform/cloud-functions/output.tf @@ -0,0 +1,11 @@ +output "cloud-function-trigger-url" { + value = google_cloudfunctions_function.cloud_function.https_trigger_url +} + +output "cloud-function-name" { + value = google_cloudfunctions_function.cloud_function.name +} + +output "cloud-function-region" { + value = google_cloudfunctions_function.cloud_function.region +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/cloud-functions/variables.tf b/learning/tour-of-beam/terraform/cloud-functions/variables.tf new file mode 100644 index 000000000000..136535b06765 --- /dev/null +++ b/learning/tour-of-beam/terraform/cloud-functions/variables.tf @@ -0,0 +1,7 @@ +variable "project_id" { + description = "The GCP Project ID where ToB Cloud Functions will be created" +} + +variable "region" { + description = "Region of App" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/iam.tf b/learning/tour-of-beam/terraform/iam.tf deleted file mode 100644 index 1236ab5c8619..000000000000 --- a/learning/tour-of-beam/terraform/iam.tf +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#resource "google_service_account" "sa_cloud_function" { -# account_id = var.service_account_name -# display_name = "Service Account to run Cloud Functions" -# project = var.project_id -#} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf index 68c8898ec7a3..1fda102f7e89 100644 --- a/learning/tour-of-beam/terraform/main.tf +++ b/learning/tour-of-beam/terraform/main.tf @@ -1,68 +1,24 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -provider "google" { - project = var.project_id - region = var.region +module "setup" { + depends_on = [module.setup] + source = "./setup" + project_id = var.project_id + service_account_id = var.service_account_id } -resource "google_cloudfunctions_function" "backend_function_1" { - name = "getSdkList-function" - runtime = "go116" - region = var.region - service_account_email = var.service_account_email - ingress_settings = "ALLOW_ALL" - # Get the source code of the cloud function as a Zip compression - source_archive_bucket = google_storage_bucket.function_bucket.name - source_archive_object = google_storage_bucket_object.zip.name - - trigger_http = true - # Name of the function that will be executed when the Google Cloud Function is triggered (def hello_gcs) - entry_point = "getSdkList" +module "buckets" { + depends_on = [module.setup] + source = "./buckets" + project_id = var.project_id + name = var.function_bucket_name + storage_class = var.function_bucket_storage_class + location = var.function_bucket_location } -resource "google_cloudfunctions_function" "backend_function_2" { - name = "getContentTree-function" - runtime = "go116" - region = var.region - service_account_email = var.service_account_email - ingress_settings = "ALLOW_ALL" - # Get the source code of the cloud function as a Zip compression - source_archive_bucket = google_storage_bucket.function_bucket.name - source_archive_object = google_storage_bucket_object.zip.name - - trigger_http = true - # Name of the function that will be executed when the Google Cloud Function is triggered (def hello_gcs) - entry_point = "getContentTree" - -} - -resource "google_cloudfunctions_function" "backend_function_3" { - name = "getUnitContent-function" - runtime = "go116" - region = var.region - service_account_email = var.service_account_email - ingress_settings = "ALLOW_ALL" - # Get the source code of the cloud function as a Zip compression - source_archive_bucket = google_storage_bucket.function_bucket.name - source_archive_object = google_storage_bucket_object.zip.name - - trigger_http = true - # Name of the function that will be executed when the Google Cloud Function is triggered (def hello_gcs) - entry_point = "getUnitContent" - -} +module "cloud-functions" { + depends_on = [module.buckets, module.setup] + source = "./cloud-functions" + project_id = var.project_id + service_account_email = module.setup.service-account-email + region = var.region +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/output.tf b/learning/tour-of-beam/terraform/output.tf new file mode 100644 index 000000000000..d77e377b6823 --- /dev/null +++ b/learning/tour-of-beam/terraform/output.tf @@ -0,0 +1,7 @@ +output "function-bucket-id" { + value = module.buckets.function-bucket-id +} + +output "cloud-function-trigger-url" { + value = module.cloud-functions.cloud-function-trigger-url +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/outputs.tf b/learning/tour-of-beam/terraform/outputs.tf deleted file mode 100644 index 9800ed8c7e56..000000000000 --- a/learning/tour-of-beam/terraform/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Atpache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -output "cloud_function_trigger_url_1" { - value = google_cloudfunctions_function.backend_function_1.https_trigger_url -} - -output "cloud_function_trigger_url_2" { - value = google_cloudfunctions_function.backend_function_2.https_trigger_url -} - -output "cloud_function_trigger_url_3" { - value = google_cloudfunctions_function.backend_function_3.https_trigger_url -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/provider.tf b/learning/tour-of-beam/terraform/provider.tf new file mode 100644 index 000000000000..e91570b1793a --- /dev/null +++ b/learning/tour-of-beam/terraform/provider.tf @@ -0,0 +1,19 @@ +terraform { + # this describe buket for save state playground cloud + backend "gcs" { + } + + required_providers { + google = { + source = "hashicorp/google" + version = "4.0.0" + } + } +} + +provider "google" { + region = var.region + project = var.project_id + // TODO may need to run module.setup first independent of this solution and add the terraform service account as a variable + // This allows us to use a service account to provision resources without downloading or storing service account keys +} diff --git a/learning/tour-of-beam/terraform/setup/iam.tf b/learning/tour-of-beam/terraform/setup/iam.tf new file mode 100644 index 000000000000..3bf330ee188b --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/iam.tf @@ -0,0 +1,27 @@ +module "cloud-functions" { + source = "../cloud-functions" +} + +resource "google_service_account" "sa_cloud_function" { + account_id = var.service_account_id + display_name = "Service Account to run Cloud Functions" + project = var.project_id +} + +resource "google_project_iam_member" "terraform_service_account_roles" { + for_each = toset([ + "roles/cloudfunctions.developer", "roles/storage.objectViewer", "roles/storage.objectCreator", + ]) + role = each.key + member = "serviceAccount:${google_service_account.sa_cloud_function.email}" + project = var.project_id +} + +resource "google_cloudfunctions_function_iam_member" "invoker" { + project = var.project_id + region = module.cloud-functions.cloud-function-region + cloud_function = module.cloud-functions.cloud-function-name + + role = "roles/cloudfunctions.invoker" + member = "allUsers" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/output.tf b/learning/tour-of-beam/terraform/setup/output.tf new file mode 100644 index 000000000000..cf9c64097f91 --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/output.tf @@ -0,0 +1,3 @@ +output "service-account-email" { + value = google_service_account.sa_cloud_function.email +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/services.tf b/learning/tour-of-beam/terraform/setup/services.tf new file mode 100644 index 000000000000..887baef74cc9 --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/services.tf @@ -0,0 +1,23 @@ +# Enable API for Cloud Build +resource "google_project_service" "cloud_build" { + project = var.project_id + service = "cloudbuild.googleapis.com" +} + +# Enable API for Cloud Function +resource "google_project_service" "cloud_function" { + project = var.project_id + service = "cloudfunctions.googleapis.com" +} + +# Enable API for Resource Manager +resource "google_project_service" "resource_manager" { + project = var.project_id + service = "cloudresourcemanager.googleapis.com" +} + +# Enable API for IAM +resource "google_project_service" "iam" { + project = var.project_id + service = "iam.googleapis.com" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/variables.tf b/learning/tour-of-beam/terraform/setup/variables.tf new file mode 100644 index 000000000000..f5ccf641c91f --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/variables.tf @@ -0,0 +1,9 @@ +variable "project_id" { + description = "The GCP Project ID where ToB Cloud Functions will be created" +} + +variable "service_account_id" { + description = "Service account ID" + default = "tour-of-beam-sa" +} + diff --git a/learning/tour-of-beam/terraform/src/main.py b/learning/tour-of-beam/terraform/src/main.py deleted file mode 100644 index 5c3c33f5dd9c..000000000000 --- a/learning/tour-of-beam/terraform/src/main.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import logging - -def http_handler(request): - """ - HTTP Cloud Function that returns the value of FOO. - Args: - request (flask.Request): The request object. - - Returns: - The response text, or any set of values that can be turned into a - Response object using `make_response` - . - """ - - logging.info('Handler Started!') - - response_body = f'This is a message for Cloud Function HTTP Trigger' - return response_body \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/src/requirements.txt b/learning/tour-of-beam/terraform/src/requirements.txt deleted file mode 100644 index 8b137891791f..000000000000 --- a/learning/tour-of-beam/terraform/src/requirements.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/learning/tour-of-beam/terraform/storage.tf b/learning/tour-of-beam/terraform/storage.tf deleted file mode 100644 index 1afa0ab27bef..000000000000 --- a/learning/tour-of-beam/terraform/storage.tf +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -#Generates archive of source code -data "archive_file" "source" { - type = "zip" - source_dir = "../backend" - output_path = "/tmp/function.zip" -} - -#Declaring Google Cloud Storage bucket to store the code of the Cloud Function -resource "google_storage_bucket" "function_bucket" { - name = "${var.project_id}-function-bucket" - location = var.region -} - -# Add source code zip to the Cloud Function's bucket -resource "google_storage_bucket_object" "zip" { - # Use an MD5 here. If there's no changes to the source code, this won't change either. - # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing - # a redeployment when it has! - name = "${data.archive_file.source.output_md5}.zip" - bucket = google_storage_bucket.function_bucket.name - source = data.archive_file.source.output_path - content_type = "application/zip" -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf index 05ece0042316..9bcedf386afe 100644 --- a/learning/tour-of-beam/terraform/variables.tf +++ b/learning/tour-of-beam/terraform/variables.tf @@ -1,42 +1,33 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# +#COMMON variable "project_id" { description = "The GCP Project ID where ToB Cloud Functions will be created" } variable "region" { - description = "Location of App" + description = "Infrastructure Region" } -#variable "cloud_function_name" { -# description = "Name of cloud function" -#} +#IAM -#variable "runtime_name" { -# description = "Name of runtime (e.g. nodejs16, python39, dotnet3, go116, java11, ruby30, php74) see https://cloud.google.com/functions/docs/concepts/execution-environment#runtimes for more runtimes" -#} +variable "service_account_id" { + description = "Service account ID" + default = "tour-of-beam-sa" +} + +#Buckets -#variable "code_function_name" { -# description = "Name of function in code" -#} +variable "function_bucket_name" { + description = "Name of Bucket to Store Cloud Functions" + default = "tour-of-beam-cloudfunction-bucket" +} + +variable "function_bucket_location" { + description = "Cloud Functions Bucket Region" + default = "US" +} -variable "service_account_email" { - description = "Name of the service account to run Cloud Function" +variable "function_bucket_storage_class" { + description = "Functions Bucket Storage Class" + default = "STANDARD" } \ No newline at end of file From a8b220238b92076d58631950cf5e5b57d6b09553 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 12 Oct 2022 23:56:51 -0700 Subject: [PATCH 131/269] New Terraform CF backend scripts --- .../tour-of-beam/terraform-v2/cloudbuild.yaml | 48 +++++++++++++++++++ .../terraform-v2/modules/iam/main.tf | 14 ++++++ .../terraform-v2/modules/iam/outputs.tf | 3 ++ .../terraform-v2/modules/iam/variables.tf | 4 ++ learning/tour-of-beam/terraform/setup/iam.tf | 20 ++++---- 5 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 learning/tour-of-beam/terraform-v2/cloudbuild.yaml create mode 100644 learning/tour-of-beam/terraform-v2/modules/iam/main.tf create mode 100644 learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf create mode 100644 learning/tour-of-beam/terraform-v2/modules/iam/variables.tf diff --git a/learning/tour-of-beam/terraform-v2/cloudbuild.yaml b/learning/tour-of-beam/terraform-v2/cloudbuild.yaml new file mode 100644 index 000000000000..9ddf29d1a587 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/cloudbuild.yaml @@ -0,0 +1,48 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - id: 'branch name' + name: 'alpine' + entrypoint: 'sh' + args: + - '-c' + - | + echo "***********************" + echo "$BRANCH_NAME" + echo "***********************" + + - id: 'tf init' + name: 'hashicorp/terraform:1.3.1' + entrypoint: 'sh' + args: + - '-c' + - | + if [ -d "environments/$BRANCH_NAME/" ]; then + cd environments/$BRANCH_NAME + terraform init + else + for dir in environments/*/ + do + cd ${dir} + env=${dir%*/} + env=${env#*/} + echo "" + echo "*************** TERRAFORM INIT ******************" + echo "******* At environment: ${env} ********" + echo "*************************************************" + terraform init || exit 1 + cd ../../ + done + fi diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf new file mode 100644 index 000000000000..b0e630a55d1c --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -0,0 +1,14 @@ +resource "google_service_account" "sa_cloud_function" { + account_id = var.service_account_id + display_name = "Service Account to run Cloud Functions" + project = var.project_id +} + +resource "google_project_iam_member" "terraform_service_account_roles" { + for_each = toset([ + "roles/cloudfunctions.developer", "roles/storage.objectViewer", "roles/storage.objectCreator", + ]) + role = each.key + member = "serviceAccount:${google_service_account.sa_cloud_function.email}" + project = var.project_id +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf new file mode 100644 index 000000000000..cf9c64097f91 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf @@ -0,0 +1,3 @@ +output "service-account-email" { + value = google_service_account.sa_cloud_function.email +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf new file mode 100644 index 000000000000..0219f0d496a9 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -0,0 +1,4 @@ +variable "project_id" {} + +variable "service_account_id" {} + diff --git a/learning/tour-of-beam/terraform/setup/iam.tf b/learning/tour-of-beam/terraform/setup/iam.tf index 3bf330ee188b..835add0e2468 100644 --- a/learning/tour-of-beam/terraform/setup/iam.tf +++ b/learning/tour-of-beam/terraform/setup/iam.tf @@ -1,7 +1,3 @@ -module "cloud-functions" { - source = "../cloud-functions" -} - resource "google_service_account" "sa_cloud_function" { account_id = var.service_account_id display_name = "Service Account to run Cloud Functions" @@ -17,11 +13,11 @@ resource "google_project_iam_member" "terraform_service_account_roles" { project = var.project_id } -resource "google_cloudfunctions_function_iam_member" "invoker" { - project = var.project_id - region = module.cloud-functions.cloud-function-region - cloud_function = module.cloud-functions.cloud-function-name - - role = "roles/cloudfunctions.invoker" - member = "allUsers" -} \ No newline at end of file +#resource "google_cloudfunctions_function_iam_member" "invoker" { +# project = var.project_id +# region = module.cloud-functions.cloud-function-region +# cloud_function = module.cloud-functions.cloud-function-name +# +# role = "roles/cloudfunctions.invoker" +# member = "allUsers" +#} \ No newline at end of file From b424b078d08116aa8784ac66c512c5ed9b128df0 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 00:17:10 -0700 Subject: [PATCH 132/269] Update Terraform v2 for backend --- .../terraform-v2/environments/uat/backend.tf | 6 +++++ .../terraform-v2/environments/uat/main.tf | 19 ++++++++++++++++ .../terraform-v2/environments/uat/outputs.tf | 11 ++++++++++ .../environments/uat/variables.tf | 6 +++++ .../terraform-v2/modules/buckets/main.tf | 22 +++++++++++++++++++ .../terraform-v2/modules/buckets/outputs.tf | 7 ++++++ .../terraform-v2/modules/buckets/variables.tf | 14 ++++++++++++ .../terraform-v2/modules/iam/variables.tf | 8 +++++-- .../tour-of-beam/terraform/buckets/main.tf | 7 ------ 9 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 learning/tour-of-beam/terraform-v2/environments/uat/backend.tf create mode 100644 learning/tour-of-beam/terraform-v2/environments/uat/main.tf create mode 100644 learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf create mode 100644 learning/tour-of-beam/terraform-v2/environments/uat/variables.tf create mode 100644 learning/tour-of-beam/terraform-v2/modules/buckets/main.tf create mode 100644 learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf create mode 100644 learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/backend.tf b/learning/tour-of-beam/terraform-v2/environments/uat/backend.tf new file mode 100644 index 000000000000..98d17640f35f --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/environments/uat/backend.tf @@ -0,0 +1,6 @@ +terraform { + backend "gcs" { + bucket = var.bucket_name + prefix = "env/uat" + } +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf new file mode 100644 index 000000000000..857c8577b713 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -0,0 +1,19 @@ +locals { + env = "uat" +} + +provider "google" { + project = var.project_id +} + +module "iam" { + source = "../../modules/iam" +} + +module "buckets" { + source = "../../modules/buckets" +} + +#module "api_enable" { +# source = "../../modules/api_enable" +#} diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf b/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf new file mode 100644 index 000000000000..fddd1294f158 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf @@ -0,0 +1,11 @@ +output "service-account-email" { + value = module.iam.service-account-email +} + +output "tf-state-bucket-id" { + value = module.buckets.tf-state-bucket-id +} + +output "tf-state-bucket-name" { + value = module.buckets.tf-state-bucket-name +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf new file mode 100644 index 000000000000..449ef2dc16be --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -0,0 +1,6 @@ +variable "bucket_name" { + description = "Bucket name to store TF State" +} +variable "project_id" { + description = "Our GCP Project" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf new file mode 100644 index 000000000000..9bac0f9a2be0 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -0,0 +1,22 @@ +resource "google_storage_bucket" "tf_state_bucket" { + name = var.name + location = var.location + project = var.project_id + storage_class = "STANDARD" +} + +#resource "google_storage_bucket_object" "zip" { +# # Use an MD5 here. If there's no changes to the source code, this won't change either. +# # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing +# # a redeployment when it has! +# name = "${data.archive_file.source.output_md5}.zip" +# bucket = google_storage_bucket.function_bucket.name +# source = data.archive_file.source.output_path +# content_type = "application/zip" +#} +# +#resource "google_storage_bucket_access_control" "public_rule" { +# bucket = google_storage_bucket.function_bucket.name +# role = "READER" +# entity = "allUsers" +#} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf new file mode 100644 index 000000000000..6ea2bdac03da --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf @@ -0,0 +1,7 @@ +output "tf-state-bucket-id" { + value = google_storage_bucket.tf_state_bucket.id +} + +output "tf-state-bucket-name" { + value = google_storage_bucket.tf_state_bucket.name +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf new file mode 100644 index 000000000000..a4f1cd3d0a4d --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -0,0 +1,14 @@ +#Generates archive of source code +variable "name" { + description = "Bucket name to store TF state" + default = "tour-of-beam-backend-tfstate-bucket" +} + +variable "location" { + description = "Cloud Functions Bucket Region" + default = "europe-west1" +} + +variable "project_id" { + description = "Our GCP Project" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf index 0219f0d496a9..ae6eff9fca13 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -1,4 +1,8 @@ -variable "project_id" {} +variable "project_id" { + description = "Our GCP Project" +} -variable "service_account_id" {} +variable "service_account_id" { + description = "Name of SA" +} diff --git a/learning/tour-of-beam/terraform/buckets/main.tf b/learning/tour-of-beam/terraform/buckets/main.tf index b663e73177da..c113197b5844 100644 --- a/learning/tour-of-beam/terraform/buckets/main.tf +++ b/learning/tour-of-beam/terraform/buckets/main.tf @@ -1,10 +1,3 @@ -#Generates archive of source code -data "archive_file" "source" { - type = "zip" - source_dir = "../../backend" - output_path = "/tmp/backend.zip" -} - resource "google_storage_bucket" "function_bucket" { name = var.name location = var.location From 308d7bd5e8bceaf4d34f5c615d34a93827eddfb3 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 00:18:28 -0700 Subject: [PATCH 133/269] Update backend.tf --- learning/tour-of-beam/terraform-v2/environments/uat/backend.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/backend.tf b/learning/tour-of-beam/terraform-v2/environments/uat/backend.tf index 98d17640f35f..4131f13934e5 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/backend.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/backend.tf @@ -1,6 +1,6 @@ terraform { backend "gcs" { - bucket = var.bucket_name + bucket = "tour-of-beam-backend-tfstate-bucket" prefix = "env/uat" } } \ No newline at end of file From 3c914034d3781ead95df46ebf9b64d3a219eab2d Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 00:25:34 -0700 Subject: [PATCH 134/269] Added api_enable tf scripts --- .../tour-of-beam/terraform-v2/cloudbuild.yaml | 43 +++++++++++++++++++ .../terraform-v2/modules/api_enable/main.tf | 23 ++++++++++ .../modules/api_enable/outputs.tf | 0 .../modules/api_enable/variables.tf | 1 + 4 files changed, 67 insertions(+) create mode 100644 learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf create mode 100644 learning/tour-of-beam/terraform-v2/modules/api_enable/outputs.tf create mode 100644 learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf diff --git a/learning/tour-of-beam/terraform-v2/cloudbuild.yaml b/learning/tour-of-beam/terraform-v2/cloudbuild.yaml index 9ddf29d1a587..57f36ee00bca 100644 --- a/learning/tour-of-beam/terraform-v2/cloudbuild.yaml +++ b/learning/tour-of-beam/terraform-v2/cloudbuild.yaml @@ -46,3 +46,46 @@ steps: cd ../../ done fi + + # [START tf-plan] + - id: 'tf plan' + name: 'hashicorp/terraform:1.3.1' + entrypoint: 'sh' + args: + - '-c' + - | + if [ -d "environments/$BRANCH_NAME/" ]; then + cd environments/$BRANCH_NAME + terraform plan + else + for dir in environments/*/ + do + cd ${dir} + env=${dir%*/} + env=${env#*/} + echo "" + echo "*************** TERRAFOM PLAN ******************" + echo "******* At environment: ${env} ********" + echo "*************************************************" + terraform plan || exit 1 + cd ../../ + done + fi + # [END tf-plan] + + # [START tf-apply] + - id: 'tf apply' + name: 'hashicorp/terraform:1.3.1' + entrypoint: 'sh' + args: + - '-c' + - | + if [ -d "environments/$BRANCH_NAME/" ]; then + cd environments/$BRANCH_NAME + terraform apply -auto-approve + else + echo "***************************** SKIPPING APPLYING *******************************" + echo "Branch '$BRANCH_NAME' does not represent an official environment." + echo "*******************************************************************************" + fi +# [END tf-apply] \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf new file mode 100644 index 000000000000..887baef74cc9 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf @@ -0,0 +1,23 @@ +# Enable API for Cloud Build +resource "google_project_service" "cloud_build" { + project = var.project_id + service = "cloudbuild.googleapis.com" +} + +# Enable API for Cloud Function +resource "google_project_service" "cloud_function" { + project = var.project_id + service = "cloudfunctions.googleapis.com" +} + +# Enable API for Resource Manager +resource "google_project_service" "resource_manager" { + project = var.project_id + service = "cloudresourcemanager.googleapis.com" +} + +# Enable API for IAM +resource "google_project_service" "iam" { + project = var.project_id + service = "iam.googleapis.com" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/outputs.tf new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf new file mode 100644 index 000000000000..6911556306ea --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf @@ -0,0 +1 @@ +variable "project_id" {} \ No newline at end of file From 8b751a20e1d7dc398ffbd64823639abb1ade916b Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 00:26:18 -0700 Subject: [PATCH 135/269] Update main.tf --- learning/tour-of-beam/terraform-v2/environments/uat/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index 857c8577b713..8e912c34b5fd 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -14,6 +14,6 @@ module "buckets" { source = "../../modules/buckets" } -#module "api_enable" { -# source = "../../modules/api_enable" -#} +module "api_enable" { + source = "../../modules/api_enable" +} From 9c68685de78783271d875c6c27a7bb6d6a0b3b1f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 00:28:26 -0700 Subject: [PATCH 136/269] Update main.tf --- learning/tour-of-beam/terraform-v2/environments/uat/main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index 8e912c34b5fd..1bc048186175 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -8,12 +8,15 @@ provider "google" { module "iam" { source = "../../modules/iam" + project_id = var.project_id } module "buckets" { source = "../../modules/buckets" + project_id = var.project_id } module "api_enable" { source = "../../modules/api_enable" + project_id = var.project_id } From ed25e54659af4fd392c4103082cb10e180f4c1c5 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 00:31:28 -0700 Subject: [PATCH 137/269] Updated tf files --- learning/tour-of-beam/terraform-v2/environments/uat/main.tf | 3 +++ .../tour-of-beam/terraform-v2/environments/uat/variables.tf | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index 1bc048186175..6b50e12ea608 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -9,14 +9,17 @@ provider "google" { module "iam" { source = "../../modules/iam" project_id = var.project_id + service_account_id = var.service_account } module "buckets" { source = "../../modules/buckets" project_id = var.project_id + service_account_id = var.service_account } module "api_enable" { source = "../../modules/api_enable" project_id = var.project_id + service_account_id = var.service_account } diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index 449ef2dc16be..8e6a509a7d8b 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -3,4 +3,6 @@ variable "bucket_name" { } variable "project_id" { description = "Our GCP Project" -} \ No newline at end of file +} + +variable "service_account" {} \ No newline at end of file From f66ee33b33eae5b97bb0e439685f8ba22cb4fb58 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 00:33:26 -0700 Subject: [PATCH 138/269] Update main.tf --- learning/tour-of-beam/terraform-v2/environments/uat/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index 6b50e12ea608..85cf879cd126 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -15,11 +15,9 @@ module "iam" { module "buckets" { source = "../../modules/buckets" project_id = var.project_id - service_account_id = var.service_account } module "api_enable" { source = "../../modules/api_enable" project_id = var.project_id - service_account_id = var.service_account } From 03c3626ddd8db2159e5cab30e227658b33d4c689 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 00:38:36 -0700 Subject: [PATCH 139/269] Update main.tf --- .../terraform-v2/modules/buckets/main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf index 9bac0f9a2be0..d32f1ae6fb49 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -1,9 +1,9 @@ -resource "google_storage_bucket" "tf_state_bucket" { - name = var.name - location = var.location - project = var.project_id - storage_class = "STANDARD" -} +#resource "google_storage_bucket" "tf_state_bucket" { +# name = var.name +# location = var.location +# project = var.project_id +# storage_class = "STANDARD" +#} #resource "google_storage_bucket_object" "zip" { # # Use an MD5 here. If there's no changes to the source code, this won't change either. From bec03ed658a308320ca89fb1e6e800bdff3b8284 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 00:41:53 -0700 Subject: [PATCH 140/269] Updated some tf sciprts --- .../terraform-v2/environments/uat/outputs.tf | 14 +++++++------- .../terraform-v2/modules/buckets/outputs.tf | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf b/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf index fddd1294f158..cf94edb11ef3 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf @@ -2,10 +2,10 @@ output "service-account-email" { value = module.iam.service-account-email } -output "tf-state-bucket-id" { - value = module.buckets.tf-state-bucket-id -} - -output "tf-state-bucket-name" { - value = module.buckets.tf-state-bucket-name -} \ No newline at end of file +#output "tf-state-bucket-id" { +# value = module.buckets.tf-state-bucket-id +#} +# +#output "tf-state-bucket-name" { +# value = module.buckets.tf-state-bucket-name +#} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf index 6ea2bdac03da..ba7612e931ff 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf @@ -1,7 +1,7 @@ -output "tf-state-bucket-id" { - value = google_storage_bucket.tf_state_bucket.id -} - -output "tf-state-bucket-name" { - value = google_storage_bucket.tf_state_bucket.name -} \ No newline at end of file +#output "tf-state-bucket-id" { +# value = google_storage_bucket.tf_state_bucket.id +#} +# +#output "tf-state-bucket-name" { +# value = google_storage_bucket.tf_state_bucket.name +#} \ No newline at end of file From 46294122b84f5a8893bf44b277d17ae13ff18908 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 01:14:52 -0700 Subject: [PATCH 141/269] Updated TF Files v.2 --- .../terraform-v2/environments/uat/main.tf | 7 ++++ .../terraform-v2/environments/uat/outputs.tf | 26 ++++++++---- .../terraform-v2/modules/buckets/main.tf | 42 +++++++++---------- .../terraform-v2/modules/buckets/outputs.tf | 18 ++++---- .../terraform-v2/modules/buckets/variables.tf | 9 +++- .../modules/cloud_functions/main.tf | 24 +++++++++++ .../modules/cloud_functions/outputs.tf | 7 ++++ .../modules/cloud_functions/variables.tf | 2 + 8 files changed, 98 insertions(+), 37 deletions(-) create mode 100644 learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf create mode 100644 learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf create mode 100644 learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index 85cf879cd126..2cc6f66768cc 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -10,6 +10,7 @@ module "iam" { source = "../../modules/iam" project_id = var.project_id service_account_id = var.service_account + depends_on = [module.buckets] } module "buckets" { @@ -20,4 +21,10 @@ module "buckets" { module "api_enable" { source = "../../modules/api_enable" project_id = var.project_id + depends_on = [module.buckets, module.iam] } + +module "cloud_functions" { + source = "../../modules/cloud_functions" + depends_on = [module.buckets, module.iam, module.api_enable] +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf b/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf index cf94edb11ef3..007249c9120c 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf @@ -2,10 +2,22 @@ output "service-account-email" { value = module.iam.service-account-email } -#output "tf-state-bucket-id" { -# value = module.buckets.tf-state-bucket-id -#} -# -#output "tf-state-bucket-name" { -# value = module.buckets.tf-state-bucket-name -#} \ No newline at end of file +output "cloud-function-trigger-url" { + value = module.cloud_functions.cloud-function-trigger-url +} + +output "cloud-function-name" { + value = module.cloud_functions.cloud-function-name +} + +output "functions-bucket-id" { + value = module.buckets.functions-bucket-id +} + +output "functions-bucket-name" { + value = module.buckets.functions-bucket-name +} + +output "function-bucket-object" { + value = module.buckets.function-bucket-object +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf index d32f1ae6fb49..8ff23f8aa505 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -1,22 +1,22 @@ -#resource "google_storage_bucket" "tf_state_bucket" { -# name = var.name -# location = var.location -# project = var.project_id -# storage_class = "STANDARD" -#} +resource "google_storage_bucket" "functions_bucket" { + name = var.name + location = var.location + project = var.project_id + storage_class = "STANDARD" +} -#resource "google_storage_bucket_object" "zip" { -# # Use an MD5 here. If there's no changes to the source code, this won't change either. -# # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing -# # a redeployment when it has! -# name = "${data.archive_file.source.output_md5}.zip" -# bucket = google_storage_bucket.function_bucket.name -# source = data.archive_file.source.output_path -# content_type = "application/zip" -#} -# -#resource "google_storage_bucket_access_control" "public_rule" { -# bucket = google_storage_bucket.function_bucket.name -# role = "READER" -# entity = "allUsers" -#} \ No newline at end of file +resource "google_storage_bucket_object" "zip" { + # Use an MD5 here. If there's no changes to the source code, this won't change either. + # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing + # a redeployment when it has! + name = "${data.archive_file.source.output_md5}.zip" + bucket = google_storage_bucket.functions_bucket.name + source = data.archive_file.source.output_path + content_type = "application/zip" +} + +resource "google_storage_bucket_access_control" "public_rule" { + bucket = google_storage_bucket.functions_bucket.name + role = "READER" + entity = "allUsers" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf index ba7612e931ff..bffafd3c2d56 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf @@ -1,7 +1,11 @@ -#output "tf-state-bucket-id" { -# value = google_storage_bucket.tf_state_bucket.id -#} -# -#output "tf-state-bucket-name" { -# value = google_storage_bucket.tf_state_bucket.name -#} \ No newline at end of file +output "functions-bucket-id" { + value = google_storage_bucket.functions_bucket.id +} + +output "functions-bucket-name" { + value = google_storage_bucket.functions_bucket.name +} + +output "function-bucket-object" { + value = google_storage_bucket_object.zip.name +} diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf index a4f1cd3d0a4d..6503f0d178ca 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -1,7 +1,6 @@ #Generates archive of source code variable "name" { - description = "Bucket name to store TF state" - default = "tour-of-beam-backend-tfstate-bucket" + description = "Bucket name to store function source code" } variable "location" { @@ -11,4 +10,10 @@ variable "location" { variable "project_id" { description = "Our GCP Project" +} + +data "archive_file" "source" { + type = "zip" + source_dir = "../../../backend" + output_path = "/tmp/backend.zip" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf new file mode 100644 index 000000000000..52d4df1a7bb9 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -0,0 +1,24 @@ +resource "google_cloudfunctions_function" "cloud_function" { + name = "tour-of-beam-backend-cloud-function" + runtime = "go116" + project = var.project_id + region = var.region + service_account_email = module.iam.service-account-email + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + source_archive_bucket = module.buckets.functions-bucket-name + source_archive_object = module.buckets.function-bucket-object + + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered + entry_point = "init" + +} + +module "iam" { + source = "../iam" +} + +module "buckets" { + source = "../buckets" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf new file mode 100644 index 000000000000..227787919121 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf @@ -0,0 +1,7 @@ +output "cloud-function-trigger-url" { + value = google_cloudfunctions_function.cloud_function.https_trigger_url +} + +output "cloud-function-name" { + value = google_cloudfunctions_function.cloud_function.name +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf new file mode 100644 index 000000000000..636a3ae98602 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -0,0 +1,2 @@ +variable "region" {} +variable "project_id" {} \ No newline at end of file From 3af4f1d56a881ddef9007a60495b7d5e1b4f713c Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 01:19:21 -0700 Subject: [PATCH 142/269] Updated files tf scripts --- learning/tour-of-beam/terraform-v2/modules/buckets/main.tf | 2 +- .../tour-of-beam/terraform-v2/modules/buckets/variables.tf | 2 +- .../terraform-v2/modules/cloud_functions/main.tf | 2 ++ .../terraform-v2/modules/cloud_functions/variables.tf | 5 ++++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf index 8ff23f8aa505..5ab4734a75c3 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -1,5 +1,5 @@ resource "google_storage_bucket" "functions_bucket" { - name = var.name + name = var.bucket_name location = var.location project = var.project_id storage_class = "STANDARD" diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf index 6503f0d178ca..0416cc20cbfd 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -1,5 +1,5 @@ #Generates archive of source code -variable "name" { +variable "bucket_name" { description = "Bucket name to store function source code" } diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 52d4df1a7bb9..24581c1573b9 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -21,4 +21,6 @@ module "iam" { module "buckets" { source = "../buckets" + project_id = var.project_id + name = var.bucket_name } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf index 636a3ae98602..125389d4c4b6 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -1,2 +1,5 @@ variable "region" {} -variable "project_id" {} \ No newline at end of file +variable "project_id" {} +variable "bucket_name" { + description = "Bucket name to store function source code" +} \ No newline at end of file From ec90bb4e406bd53f0de3fccccf397619d298b91e Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 01:20:09 -0700 Subject: [PATCH 143/269] Update main.tf --- learning/tour-of-beam/terraform-v2/environments/uat/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index 2cc6f66768cc..9f6537315852 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -16,6 +16,7 @@ module "iam" { module "buckets" { source = "../../modules/buckets" project_id = var.project_id + bucket_name = var.bucket_name } module "api_enable" { From 2650851ffa017d87142fc09eee480c41c4fed8d2 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 01:21:17 -0700 Subject: [PATCH 144/269] Updated tf scripts --- .../tour-of-beam/terraform-v2/modules/cloud_functions/main.tf | 1 + .../terraform-v2/modules/cloud_functions/variables.tf | 3 +++ 2 files changed, 4 insertions(+) diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 24581c1573b9..326c6b806777 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -17,6 +17,7 @@ resource "google_cloudfunctions_function" "cloud_function" { module "iam" { source = "../iam" + service_account_id = var.service_account_id } module "buckets" { diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf index 125389d4c4b6..7f5f62d7b336 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -2,4 +2,7 @@ variable "region" {} variable "project_id" {} variable "bucket_name" { description = "Bucket name to store function source code" +} +variable "service_account_id" { + description = "Name of SA" } \ No newline at end of file From d215fc5acec751f2c10b68176a0c6fa832fdfa08 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 01:22:00 -0700 Subject: [PATCH 145/269] Update main.tf --- .../tour-of-beam/terraform-v2/modules/cloud_functions/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 326c6b806777..559c92f0e57c 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -23,5 +23,5 @@ module "iam" { module "buckets" { source = "../buckets" project_id = var.project_id - name = var.bucket_name + bucket_name = var.bucket_name } \ No newline at end of file From 26e729611eec81ada4fe771410595d698b77a769 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 01:22:32 -0700 Subject: [PATCH 146/269] Update main.tf --- .../tour-of-beam/terraform-v2/modules/cloud_functions/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 559c92f0e57c..3f0fe27d7a9d 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -18,6 +18,7 @@ resource "google_cloudfunctions_function" "cloud_function" { module "iam" { source = "../iam" service_account_id = var.service_account_id + project_id = var.project_id } module "buckets" { From d6a07024edabc78feb3275dcbd8db2b14acc2b53 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 01:24:06 -0700 Subject: [PATCH 147/269] Updates --- learning/tour-of-beam/terraform-v2/environments/uat/main.tf | 4 ++++ .../tour-of-beam/terraform-v2/environments/uat/variables.tf | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index 9f6537315852..48d8764c3849 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -27,5 +27,9 @@ module "api_enable" { module "cloud_functions" { source = "../../modules/cloud_functions" + region = var.region + project_id = var.project_id + bucket_name = var.bucket_name + service_account_id = var.service_account depends_on = [module.buckets, module.iam, module.api_enable] } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index 8e6a509a7d8b..0952b784c1f7 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -5,4 +5,7 @@ variable "project_id" { description = "Our GCP Project" } -variable "service_account" {} \ No newline at end of file +variable "service_account" {} +variable "region" { + default = "europe-west1" +} \ No newline at end of file From e01cacd1165f5f0933aca908244ee209c4fe00ed Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 01:28:32 -0700 Subject: [PATCH 148/269] Update main.tf --- learning/tour-of-beam/terraform-v2/modules/iam/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index b0e630a55d1c..76ef5039538d 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -6,7 +6,7 @@ resource "google_service_account" "sa_cloud_function" { resource "google_project_iam_member" "terraform_service_account_roles" { for_each = toset([ - "roles/cloudfunctions.developer", "roles/storage.objectViewer", "roles/storage.objectCreator", + "roles/cloudfunctions.developer", "roles/storage.objectViewer", "roles/storage.objectCreator", "roles/iam.serviceAccountAdmin", ]) role = each.key member = "serviceAccount:${google_service_account.sa_cloud_function.email}" From a87b2ab0f2dd069ced20d2d2b8586c098392a9dd Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 02:00:07 -0700 Subject: [PATCH 149/269] Update tf scripts --- .../tour-of-beam/terraform-v2/environments/uat/main.tf | 4 ++-- .../terraform-v2/environments/uat/variables.tf | 7 +++++-- .../terraform-v2/modules/api_enable/outputs.tf | 0 .../terraform-v2/modules/cloud_functions/variables.tf | 3 ++- learning/tour-of-beam/terraform-v2/modules/iam/main.tf | 2 +- .../tour-of-beam/terraform-v2/modules/iam/variables.tf | 2 +- 6 files changed, 11 insertions(+), 7 deletions(-) delete mode 100644 learning/tour-of-beam/terraform-v2/modules/api_enable/outputs.tf diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index 48d8764c3849..fe412a61beca 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -10,19 +10,19 @@ module "iam" { source = "../../modules/iam" project_id = var.project_id service_account_id = var.service_account - depends_on = [module.buckets] + depends_on = [module.api_enable] } module "buckets" { source = "../../modules/buckets" project_id = var.project_id bucket_name = var.bucket_name + depends_on = [module.iam, module.api_enable] } module "api_enable" { source = "../../modules/api_enable" project_id = var.project_id - depends_on = [module.buckets, module.iam] } module "cloud_functions" { diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index 0952b784c1f7..84b5a775b1f8 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -1,11 +1,14 @@ variable "bucket_name" { - description = "Bucket name to store TF State" + description = "Bucket name to store function source code" } variable "project_id" { description = "Our GCP Project" } -variable "service_account" {} +variable "service_account" { + description = "Name of SA to run Cloud Function" +} + variable "region" { default = "europe-west1" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/outputs.tf deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf index 7f5f62d7b336..2e3194fc6d83 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -3,6 +3,7 @@ variable "project_id" {} variable "bucket_name" { description = "Bucket name to store function source code" } + variable "service_account_id" { - description = "Name of SA" + description = "Name of SA to run Cloud Function" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index 76ef5039538d..b0e630a55d1c 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -6,7 +6,7 @@ resource "google_service_account" "sa_cloud_function" { resource "google_project_iam_member" "terraform_service_account_roles" { for_each = toset([ - "roles/cloudfunctions.developer", "roles/storage.objectViewer", "roles/storage.objectCreator", "roles/iam.serviceAccountAdmin", + "roles/cloudfunctions.developer", "roles/storage.objectViewer", "roles/storage.objectCreator", ]) role = each.key member = "serviceAccount:${google_service_account.sa_cloud_function.email}" diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf index ae6eff9fca13..8ef947b0f3ed 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -3,6 +3,6 @@ variable "project_id" { } variable "service_account_id" { - description = "Name of SA" + description = "Name of SA to run Cloud Function" } From 6eb395d23f6bfccc951c508a9951455a01de258d Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 03:10:47 -0700 Subject: [PATCH 150/269] Update main.tf --- learning/tour-of-beam/terraform-v2/modules/iam/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index b0e630a55d1c..3bbdb5de916d 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -6,7 +6,8 @@ resource "google_service_account" "sa_cloud_function" { resource "google_project_iam_member" "terraform_service_account_roles" { for_each = toset([ - "roles/cloudfunctions.developer", "roles/storage.objectViewer", "roles/storage.objectCreator", + "roles/resourcemanager.projectIamAdmin", "roles/cloudfunctions.developer", "roles/storage.objectViewer", + "roles/storage.objectCreator", "roles/iam.serviceAccountAdmin", ]) role = each.key member = "serviceAccount:${google_service_account.sa_cloud_function.email}" From 56bc7a5ad7119dacf761347023452fd1e0996e56 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 05:44:41 -0700 Subject: [PATCH 151/269] Set up GitHub workflow --- cloud-build/.dockerignore | 4 + cloud-build/.github/workflows/cloud-build.yml | 63 +++ cloud-build/Dockerfile | 34 ++ cloud-build/README.md | 88 +++++ cloud-build/index.js | 28 ++ cloud-build/package-lock.json | 374 ++++++++++++++++++ cloud-build/package.json | 14 + 7 files changed, 605 insertions(+) create mode 100644 cloud-build/.dockerignore create mode 100644 cloud-build/.github/workflows/cloud-build.yml create mode 100644 cloud-build/Dockerfile create mode 100644 cloud-build/README.md create mode 100644 cloud-build/index.js create mode 100644 cloud-build/package-lock.json create mode 100644 cloud-build/package.json diff --git a/cloud-build/.dockerignore b/cloud-build/.dockerignore new file mode 100644 index 000000000000..07efc8ee2472 --- /dev/null +++ b/cloud-build/.dockerignore @@ -0,0 +1,4 @@ +Dockerfile +README.md +node_modules +npm-debug.log diff --git a/cloud-build/.github/workflows/cloud-build.yml b/cloud-build/.github/workflows/cloud-build.yml new file mode 100644 index 000000000000..6d9778acd893 --- /dev/null +++ b/cloud-build/.github/workflows/cloud-build.yml @@ -0,0 +1,63 @@ +# Copyright 2019 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Build using Cloud Build + +on: + push: + branches: + - 'main' + +env: + PROJECT_ID: ${{ secrets.RUN_PROJECT }} + SERVICE_NAME: helloworld-nodejs + +jobs: + setup-build-deploy: + name: Setup, Build, and Deploy + runs-on: ubuntu-latest + + # Add "id-token" with the intended permissions. + permissions: + contents: 'read' + id-token: 'write' + + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Configure Workload Identity Federation and generate an access token. + - id: 'auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v0' + with: + workload_identity_provider: 'projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider' + service_account: '${{ secrets.RUN_SA_EMAIL }}' + + # Alternative option - authentication via credentials json + # - id: 'auth' + # uses: 'google-github-actions/auth@v0' + # with: + # credentials_json: '${{ secrets.RUN_SA_KEY }}' + + # Setup gcloud CLI + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + + # Build and push image to Google Container Registry + - name: Build + run: |- + gcloud builds submit \ + --quiet \ + --tag "gcr.io/$PROJECT_ID/$SERVICE_NAME:$GITHUB_SHA" diff --git a/cloud-build/Dockerfile b/cloud-build/Dockerfile new file mode 100644 index 000000000000..4eee05e71058 --- /dev/null +++ b/cloud-build/Dockerfile @@ -0,0 +1,34 @@ +# Copyright 2020 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use the official lightweight Node.js 16 image. +# https://hub.docker.com/_/node +FROM node:16-slim + +# Create and change to the app directory. +WORKDIR /usr/src/app + +# Copy application dependency manifests to the container image. +# A wildcard is used to ensure both package.json AND package-lock.json are copied. +# Copying this separately prevents re-running npm install on every code change. +COPY package*.json ./ + +# Install production dependencies. +RUN npm install --only=production + +# Copy local code to the container image. +COPY . ./ + +# Run the web service on container startup. +CMD [ "npm", "start" ] diff --git a/cloud-build/README.md b/cloud-build/README.md new file mode 100644 index 000000000000..5dad3b85b56c --- /dev/null +++ b/cloud-build/README.md @@ -0,0 +1,88 @@ +# Cloud Build - GitHub Actions + +An example workflow that uses [GitHub Actions][actions] to build a +[Hello World Node.js app](index.js) container image using [Cloud Build][cloud-build]. + +This code is intended to be an _example_. You will likely need to change or +update values to match your setup. + +## Workflow description + +For pushes to the `main` branch, this workflow will: + +1. Download and configure the Google [Cloud SDK][sdk] with the provided + credentials. + +1. Build, tag, and push a container image to Google Container Registry. + + - The image is built using Cloud Build and pushed to Google Container Registry. + + - The image is available through the following tags: `latest` and first 8 of + the commit SHA. + +## Setup + +1. Create a new Google Cloud Project (or select an existing project) and + [enable the Cloud Build and Cloud Build APIs](https://console.cloud.google.com/flows/enableapi?apiid=cloudbuild.googleapis.com,run.googleapis.com). + +1. Create or reuse a GitHub repository for the example workflow: + + 1. [Create a repository](https://help.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-new-repository). + + 1. Move into the repository directory: + + ``` + $ cd + ``` + + 1. Copy the example into the repository: + + ``` + $ cp -r /github-actions/example-workflows/cloud-build/ . + ``` + +1. [Create a Google Cloud service account][create-sa] if one does not already + exist. + +1. Add the following [Cloud IAM roles][roles] to your service account: + + - [`Cloud Build Service Account`](https://cloud.google.com/cloud-build/docs/cloud-build-service-account) - allows for execution of builds on your behalf + + - `Viewer` - allows for Cloud Build log storage + +1. [Create a JSON service account key][create-key] for the service account. + +1. Add the following secrets to your repository's secrets: + + - `RUN_PROJECT`: Google Cloud project ID + + - `RUN_SA_EMAIL`: the email of the service account + + - `RUN_SA_KEY`: the content of the service account JSON file + +## Run the workflow + +1. Add and commit your changes: + + ```text + $ git add . + $ git commit -m "Set up GitHub workflow" + ``` + +1. Push to the `main` branch: + + ```text + $ git push -u origin main + ``` + +1. View the GitHub Actions Workflow by selecting the `Actions` tab at the top + of your repository on GitHub. Then click on the `Build using Cloud Build` + element to see the details. + +[actions]: https://help.github.com/en/categories/automating-your-workflow-with-github-actions +[cloud-build]: https://cloud.google.com/cloud-build/ +[create-sa]: https://cloud.google.com/iam/docs/creating-managing-service-accounts +[create-key]: https://cloud.google.com/iam/docs/creating-managing-service-account-keys +[sdk]: https://cloud.google.com/sdk +[secrets]: https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets +[roles]: https://cloud.google.com/iam/docs/granting-roles-to-service-accounts#granting_access_to_a_service_account_for_a_resource diff --git a/cloud-build/index.js b/cloud-build/index.js new file mode 100644 index 000000000000..cdc2031cd646 --- /dev/null +++ b/cloud-build/index.js @@ -0,0 +1,28 @@ +// Copyright 2020 Google, LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const express = require('express'); +const app = express(); + +app.get('/', (req, res) => { + console.log('Hello world received a request.'); + + const target = process.env.TARGET || 'World'; + res.send(`Hello ${target}!`); +}); + +const port = process.env.PORT || 8080; +app.listen(port, () => { + console.log('Hello world listening on port', port); +}); diff --git a/cloud-build/package-lock.json b/cloud-build/package-lock.json new file mode 100644 index 000000000000..b8746679fd86 --- /dev/null +++ b/cloud-build/package-lock.json @@ -0,0 +1,374 @@ +{ + "name": "helloworld", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "accepts": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", + "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "requires": { + "mime-types": "~2.1.24", + "negotiator": "0.6.2" + } + }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "body-parser": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", + "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "requires": { + "bytes": "3.1.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.7.0", + "raw-body": "2.4.0", + "type-is": "~1.6.17" + } + }, + "bytes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" + }, + "content-disposition": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", + "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "requires": { + "safe-buffer": "5.1.2" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, + "express": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", + "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "requires": { + "accepts": "~1.3.7", + "array-flatten": "1.1.1", + "body-parser": "1.19.0", + "content-disposition": "0.5.3", + "content-type": "~1.0.4", + "cookie": "0.4.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.5", + "qs": "6.7.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.1.2", + "send": "0.17.1", + "serve-static": "1.14.1", + "setprototypeof": "1.1.1", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + } + }, + "finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + } + }, + "forwarded": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, + "http-errors": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", + "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + }, + "mime-db": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", + "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==" + }, + "mime-types": { + "version": "2.1.26", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", + "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", + "requires": { + "mime-db": "1.43.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "negotiator": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", + "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "proxy-addr": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", + "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "requires": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.9.1" + } + }, + "qs": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", + "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" + }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + }, + "raw-body": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", + "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "requires": { + "bytes": "3.1.0", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "send": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", + "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.7.2", + "mime": "1.6.0", + "ms": "2.1.1", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "dependencies": { + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + } + } + }, + "serve-static": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", + "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.1" + } + }, + "setprototypeof": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", + "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, + "toidentifier": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", + "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" + }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + } + } +} diff --git a/cloud-build/package.json b/cloud-build/package.json new file mode 100644 index 000000000000..b1f81256cc76 --- /dev/null +++ b/cloud-build/package.json @@ -0,0 +1,14 @@ +{ + "name": "helloworld", + "version": "1.0.0", + "description": "Simple hello world sample in Node", + "main": "index.js", + "scripts": { + "start": "node index.js" + }, + "author": "", + "license": "Apache-2.0", + "dependencies": { + "express": "^4.17.1" + } +} From b8237a957536cc833d9fbd227dbb89d6d804f192 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 13:09:31 -0700 Subject: [PATCH 152/269] Update --- learning/tour-of-beam/terraform-v2/modules/buckets/main.tf | 6 +++--- .../tour-of-beam/terraform-v2/modules/buckets/outputs.tf | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf index 5ab4734a75c3..8bdb07c1c650 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -1,4 +1,4 @@ -resource "google_storage_bucket" "functions_bucket" { +resource "google_storage_bucket" "bucket_for_function" { name = var.bucket_name location = var.location project = var.project_id @@ -10,13 +10,13 @@ resource "google_storage_bucket_object" "zip" { # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing # a redeployment when it has! name = "${data.archive_file.source.output_md5}.zip" - bucket = google_storage_bucket.functions_bucket.name + bucket = google_storage_bucket.bucket_for_function.name source = data.archive_file.source.output_path content_type = "application/zip" } resource "google_storage_bucket_access_control" "public_rule" { - bucket = google_storage_bucket.functions_bucket.name + bucket = google_storage_bucket.bucket_for_function.name role = "READER" entity = "allUsers" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf index bffafd3c2d56..836b6d4c2e38 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf @@ -1,9 +1,9 @@ output "functions-bucket-id" { - value = google_storage_bucket.functions_bucket.id + value = google_storage_bucket.bucket_for_function.id } output "functions-bucket-name" { - value = google_storage_bucket.functions_bucket.name + value = google_storage_bucket.bucket_for_function.name } output "function-bucket-object" { From dfdee8817fcec24d91b59512089aeb68f03db9ab Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 13:12:23 -0700 Subject: [PATCH 153/269] Update main.tf --- learning/tour-of-beam/terraform-v2/modules/buckets/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf index 8bdb07c1c650..119649bbc4c1 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -3,6 +3,7 @@ resource "google_storage_bucket" "bucket_for_function" { location = var.location project = var.project_id storage_class = "STANDARD" + force_destroy = true } resource "google_storage_bucket_object" "zip" { From e9520d84b58860c221122bec91ab8640377da193 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 13:13:43 -0700 Subject: [PATCH 154/269] Update main.tf --- learning/tour-of-beam/terraform-v2/modules/buckets/main.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf index 119649bbc4c1..5ab4734a75c3 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -1,9 +1,8 @@ -resource "google_storage_bucket" "bucket_for_function" { +resource "google_storage_bucket" "functions_bucket" { name = var.bucket_name location = var.location project = var.project_id storage_class = "STANDARD" - force_destroy = true } resource "google_storage_bucket_object" "zip" { @@ -11,13 +10,13 @@ resource "google_storage_bucket_object" "zip" { # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing # a redeployment when it has! name = "${data.archive_file.source.output_md5}.zip" - bucket = google_storage_bucket.bucket_for_function.name + bucket = google_storage_bucket.functions_bucket.name source = data.archive_file.source.output_path content_type = "application/zip" } resource "google_storage_bucket_access_control" "public_rule" { - bucket = google_storage_bucket.bucket_for_function.name + bucket = google_storage_bucket.functions_bucket.name role = "READER" entity = "allUsers" } \ No newline at end of file From 7d2f49ab43bfc8dbd48014fbb1107dbdd21ae81a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 13:16:11 -0700 Subject: [PATCH 155/269] Update outputs.tf --- learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf index 836b6d4c2e38..bffafd3c2d56 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf @@ -1,9 +1,9 @@ output "functions-bucket-id" { - value = google_storage_bucket.bucket_for_function.id + value = google_storage_bucket.functions_bucket.id } output "functions-bucket-name" { - value = google_storage_bucket.bucket_for_function.name + value = google_storage_bucket.functions_bucket.name } output "function-bucket-object" { From a73c4ebd05fdfab078f565c9d3e7859f791998bf Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 13:35:04 -0700 Subject: [PATCH 156/269] Some summary --- learning/tour-of-beam/terraform-v2/modules/iam/main.tf | 1 - learning/tour-of-beam/terraform-v2/modules/iam/variables.tf | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index 3bbdb5de916d..38315a712ac7 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -1,7 +1,6 @@ resource "google_service_account" "sa_cloud_function" { account_id = var.service_account_id display_name = "Service Account to run Cloud Functions" - project = var.project_id } resource "google_project_iam_member" "terraform_service_account_roles" { diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf index 8ef947b0f3ed..07aee218cf17 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -4,5 +4,6 @@ variable "project_id" { variable "service_account_id" { description = "Name of SA to run Cloud Function" + default = "admin-sa" } From 1302ff7b2c5d32e0e9a88f1396b12f9de4fd6fd2 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 13:36:40 -0700 Subject: [PATCH 157/269] Update variables.tf --- learning/tour-of-beam/terraform-v2/environments/uat/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index 84b5a775b1f8..11f9a2895f9e 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -7,6 +7,7 @@ variable "project_id" { variable "service_account" { description = "Name of SA to run Cloud Function" + default = "admin-sa" } variable "region" { From b33496d532d262c656312fe4bc6823a652080a1a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 13:39:27 -0700 Subject: [PATCH 158/269] Updates --- .../tour-of-beam/terraform-v2/environments/uat/variables.tf | 2 +- learning/tour-of-beam/terraform-v2/modules/iam/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index 11f9a2895f9e..db827e005ebb 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -7,7 +7,7 @@ variable "project_id" { variable "service_account" { description = "Name of SA to run Cloud Function" - default = "admin-sa" + default = "tob-bckend-sa" } variable "region" { diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf index 07aee218cf17..cefe4d54f24b 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -4,6 +4,6 @@ variable "project_id" { variable "service_account_id" { description = "Name of SA to run Cloud Function" - default = "admin-sa" + default = "tob-bckend-sa" } From 2ad535ea1ae3ff46b685f0237828d935ec722cc6 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 13:45:24 -0700 Subject: [PATCH 159/269] Updated --- .../tour-of-beam/terraform-v2/environments/uat/variables.tf | 2 +- learning/tour-of-beam/terraform-v2/modules/iam/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index db827e005ebb..11f9a2895f9e 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -7,7 +7,7 @@ variable "project_id" { variable "service_account" { description = "Name of SA to run Cloud Function" - default = "tob-bckend-sa" + default = "admin-sa" } variable "region" { diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf index cefe4d54f24b..07aee218cf17 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -4,6 +4,6 @@ variable "project_id" { variable "service_account_id" { description = "Name of SA to run Cloud Function" - default = "tob-bckend-sa" + default = "admin-sa" } From d5f6449e22b4896a18191fd07bda6f2a02bf0a65 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 13:53:13 -0700 Subject: [PATCH 160/269] Udpated --- .../tour-of-beam/terraform-v2/environments/uat/variables.tf | 2 +- learning/tour-of-beam/terraform-v2/modules/iam/main.tf | 4 ++-- learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf | 2 +- learning/tour-of-beam/terraform-v2/modules/iam/variables.tf | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index 11f9a2895f9e..3185f6ad1fc6 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -7,7 +7,7 @@ variable "project_id" { variable "service_account" { description = "Name of SA to run Cloud Function" - default = "admin-sa" + default = "newnameforsa" } variable "region" { diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index 38315a712ac7..f1082c78c570 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -1,4 +1,4 @@ -resource "google_service_account" "sa_cloud_function" { +resource "google_service_account" "tourofbeam_cf_sa" { account_id = var.service_account_id display_name = "Service Account to run Cloud Functions" } @@ -9,6 +9,6 @@ resource "google_project_iam_member" "terraform_service_account_roles" { "roles/storage.objectCreator", "roles/iam.serviceAccountAdmin", ]) role = each.key - member = "serviceAccount:${google_service_account.sa_cloud_function.email}" + member = "serviceAccount:${google_service_account.tourofbeam_cf_sa.email}" project = var.project_id } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf index cf9c64097f91..0a5da6d2e0d4 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf @@ -1,3 +1,3 @@ output "service-account-email" { - value = google_service_account.sa_cloud_function.email + value = google_service_account.tourofbeam_cf_sa.email } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf index 07aee218cf17..c8637283e6e8 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -4,6 +4,6 @@ variable "project_id" { variable "service_account_id" { description = "Name of SA to run Cloud Function" - default = "admin-sa" + default = "newnameforsa" } From 2ba8e74284ad26e567ea3ef2b00d9f4b13cf927f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 13 Oct 2022 13:57:25 -0700 Subject: [PATCH 161/269] Updates --- .../tour-of-beam/terraform-v2/environments/uat/variables.tf | 2 +- learning/tour-of-beam/terraform-v2/modules/iam/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index 3185f6ad1fc6..0f83d3d1b644 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -7,7 +7,7 @@ variable "project_id" { variable "service_account" { description = "Name of SA to run Cloud Function" - default = "newnameforsa" + default = "tourofbeam-cfsa" } variable "region" { diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf index c8637283e6e8..6c7bbda2d6b1 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -4,6 +4,6 @@ variable "project_id" { variable "service_account_id" { description = "Name of SA to run Cloud Function" - default = "newnameforsa" + default = "tourofbeam-cfsa" } From 579821ea672ffb2774b653d31e4288097a0a0e41 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 01:44:34 -0700 Subject: [PATCH 162/269] Update main.tf --- .../terraform-v2/modules/cloud_functions/main.tf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 3f0fe27d7a9d..a11d43cf2d62 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -3,11 +3,10 @@ resource "google_cloudfunctions_function" "cloud_function" { runtime = "go116" project = var.project_id region = var.region - service_account_email = module.iam.service-account-email ingress_settings = "ALLOW_ALL" # Get the source code of the cloud function as a Zip compression - source_archive_bucket = module.buckets.functions-bucket-name - source_archive_object = module.buckets.function-bucket-object +# source_archive_bucket = module.buckets.functions-bucket-name +# source_archive_object = module.buckets.function-bucket-object trigger_http = true # Name of the function that will be executed when the Google Cloud Function is triggered From 4e67f8124dd0233a3d24890b7cd25f1487503a71 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 02:33:35 -0700 Subject: [PATCH 163/269] Add --- learning/tour-of-beam/terraform-v2/environments/uat/main.tf | 1 - learning/tour-of-beam/terraform-v2/environments/uat/variables.tf | 1 - 2 files changed, 2 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index fe412a61beca..b5e4380bcaad 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -29,7 +29,6 @@ module "cloud_functions" { source = "../../modules/cloud_functions" region = var.region project_id = var.project_id - bucket_name = var.bucket_name service_account_id = var.service_account depends_on = [module.buckets, module.iam, module.api_enable] } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index 0f83d3d1b644..84b5a775b1f8 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -7,7 +7,6 @@ variable "project_id" { variable "service_account" { description = "Name of SA to run Cloud Function" - default = "tourofbeam-cfsa" } variable "region" { From 8078fd472bdf11bd08bf2d540cd3967ccc8aa84d Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 02:34:26 -0700 Subject: [PATCH 164/269] Updates --- .../terraform-v2/modules/cloud_functions/main.tf | 7 ++++--- .../terraform-v2/modules/cloud_functions/variables.tf | 4 ---- .../tour-of-beam/terraform-v2/modules/iam/variables.tf | 1 - 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index a11d43cf2d62..b3f6c55eba95 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -3,10 +3,11 @@ resource "google_cloudfunctions_function" "cloud_function" { runtime = "go116" project = var.project_id region = var.region + service_account_email = module.iam.service-account-email ingress_settings = "ALLOW_ALL" # Get the source code of the cloud function as a Zip compression -# source_archive_bucket = module.buckets.functions-bucket-name -# source_archive_object = module.buckets.function-bucket-object + source_archive_bucket = module.buckets.functions-bucket-name + source_archive_object = module.buckets.function-bucket-object trigger_http = true # Name of the function that will be executed when the Google Cloud Function is triggered @@ -23,5 +24,5 @@ module "iam" { module "buckets" { source = "../buckets" project_id = var.project_id - bucket_name = var.bucket_name + bucket_name = module.buckets.functions-bucket-name } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf index 2e3194fc6d83..6376e109c5e0 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -1,9 +1,5 @@ variable "region" {} variable "project_id" {} -variable "bucket_name" { - description = "Bucket name to store function source code" -} - variable "service_account_id" { description = "Name of SA to run Cloud Function" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf index 6c7bbda2d6b1..8ef947b0f3ed 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -4,6 +4,5 @@ variable "project_id" { variable "service_account_id" { description = "Name of SA to run Cloud Function" - default = "tourofbeam-cfsa" } From e15a76cecb198f004c32fcbbd615c0b1f6915083 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 02:44:35 -0700 Subject: [PATCH 165/269] Updates --- learning/tour-of-beam/terraform-v2/environments/uat/main.tf | 1 + .../tour-of-beam/terraform-v2/environments/uat/variables.tf | 2 -- .../tour-of-beam/terraform-v2/modules/cloud_functions/main.tf | 2 +- .../terraform-v2/modules/cloud_functions/variables.tf | 3 ++- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index b5e4380bcaad..fe412a61beca 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -29,6 +29,7 @@ module "cloud_functions" { source = "../../modules/cloud_functions" region = var.region project_id = var.project_id + bucket_name = var.bucket_name service_account_id = var.service_account depends_on = [module.buckets, module.iam, module.api_enable] } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index 84b5a775b1f8..ae128a9a3928 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -4,11 +4,9 @@ variable "bucket_name" { variable "project_id" { description = "Our GCP Project" } - variable "service_account" { description = "Name of SA to run Cloud Function" } - variable "region" { default = "europe-west1" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index b3f6c55eba95..3f0fe27d7a9d 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -24,5 +24,5 @@ module "iam" { module "buckets" { source = "../buckets" project_id = var.project_id - bucket_name = module.buckets.functions-bucket-name + bucket_name = var.bucket_name } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf index 6376e109c5e0..2e0ac30d62a7 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -2,4 +2,5 @@ variable "region" {} variable "project_id" {} variable "service_account_id" { description = "Name of SA to run Cloud Function" -} \ No newline at end of file +} +variable "bucket_name" {} \ No newline at end of file From 3d2bd56660d577d66525c4eeb13cd13f070fe3f3 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 03:58:13 -0700 Subject: [PATCH 166/269] Almost Final version --- learning/tour-of-beam/terraform-v2/environments/uat/main.tf | 6 ++++-- .../tour-of-beam/terraform-v2/environments/uat/variables.tf | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf index fe412a61beca..f8ee79915e43 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/main.tf @@ -9,7 +9,7 @@ provider "google" { module "iam" { source = "../../modules/iam" project_id = var.project_id - service_account_id = var.service_account + service_account_id = var.service_account_id depends_on = [module.api_enable] } @@ -30,6 +30,8 @@ module "cloud_functions" { region = var.region project_id = var.project_id bucket_name = var.bucket_name - service_account_id = var.service_account + service_account_id = module.iam.service-account-email + source_archive_bucket = module.buckets.functions-bucket-name + source_archive_object = module.buckets.function-bucket-object depends_on = [module.buckets, module.iam, module.api_enable] } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf index ae128a9a3928..22dd45569e2c 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf @@ -4,7 +4,7 @@ variable "bucket_name" { variable "project_id" { description = "Our GCP Project" } -variable "service_account" { +variable "service_account_id" { description = "Name of SA to run Cloud Function" } variable "region" { From 2a9df59f5990fd79440b6919f05d1207c6dde387 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 03:58:46 -0700 Subject: [PATCH 167/269] Almost Final Version --- .../modules/cloud_functions/main.tf | 20 +++---------------- .../modules/cloud_functions/variables.tf | 5 ++++- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 3f0fe27d7a9d..20a1cefe258c 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -2,27 +2,13 @@ resource "google_cloudfunctions_function" "cloud_function" { name = "tour-of-beam-backend-cloud-function" runtime = "go116" project = var.project_id + service_account_email = var.service_account_id + source_archive_bucket = var.source_archive_bucket + source_archive_object = var.source_archive_object region = var.region - service_account_email = module.iam.service-account-email ingress_settings = "ALLOW_ALL" # Get the source code of the cloud function as a Zip compression - source_archive_bucket = module.buckets.functions-bucket-name - source_archive_object = module.buckets.function-bucket-object - trigger_http = true # Name of the function that will be executed when the Google Cloud Function is triggered entry_point = "init" - -} - -module "iam" { - source = "../iam" - service_account_id = var.service_account_id - project_id = var.project_id -} - -module "buckets" { - source = "../buckets" - project_id = var.project_id - bucket_name = var.bucket_name } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf index 2e0ac30d62a7..4cbec6697f73 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -3,4 +3,7 @@ variable "project_id" {} variable "service_account_id" { description = "Name of SA to run Cloud Function" } -variable "bucket_name" {} \ No newline at end of file +variable "bucket_name" {} +variable "source_archive_bucket" {} + +variable "source_archive_object" {} \ No newline at end of file From 0c5e238eeeabb3e3d2cd8c7acae4ab1f97e495ba Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 06:45:30 -0700 Subject: [PATCH 168/269] Update --- learning/tour-of-beam/backend_v2/README.md | 56 ++ .../backend_v2/cmd/ci_cd/ci_cd.go | 54 ++ learning/tour-of-beam/backend_v2/cmd/main.go | 36 + .../WEB-INF/appengine-generated/local_db.bin | Bin 0 -> 6927 bytes .../backend_v2/datadir/WEB-INF/index.yaml | 20 + .../tour-of-beam/backend_v2/datadir/env.yaml | 6 + .../backend_v2/docker-compose.yml | 28 + learning/tour-of-beam/backend_v2/function.go | 178 +++++ learning/tour-of-beam/backend_v2/go.mod | 28 + learning/tour-of-beam/backend_v2/go.sum | 688 ++++++++++++++++++ .../backend_v2/internal/entity.go | 69 ++ .../internal/fs_content/builders.go | 45 ++ .../backend_v2/internal/fs_content/load.go | 210 ++++++ .../internal/fs_content/load_test.go | 79 ++ .../backend_v2/internal/fs_content/yaml.go | 81 +++ .../tour-of-beam/backend_v2/internal/json.go | 29 + .../tour-of-beam/backend_v2/internal/sdk.go | 51 ++ .../backend_v2/internal/sdk_test.go | 59 ++ .../backend_v2/internal/service/content.go | 42 ++ .../backend_v2/internal/service/mock.go | 48 ++ .../backend_v2/internal/storage/adapter.go | 146 ++++ .../backend_v2/internal/storage/datastore.go | 260 +++++++ .../backend_v2/internal/storage/iface.go | 29 + .../internal/storage/image/Dockerfile | 32 + .../internal/storage/image/start-datastore.sh | 61 ++ .../backend_v2/internal/storage/index.yaml | 20 + .../backend_v2/internal/storage/schema.go | 99 +++ .../samples/api/get_content_tree.json | 41 ++ .../samples/api/get_unit_content.json | 10 + .../learning-content/java/content-info.yaml | 4 + .../java/module 1/module-info.yaml | 7 + .../module 1/unit-challenge/description.md | 3 + .../java/module 1/unit-challenge/hint1.md | 3 + .../java/module 1/unit-challenge/hint2.md | 3 + .../module 1/unit-challenge/unit-info.yaml | 5 + .../java/module 1/unit-example/unit-info.yaml | 3 + .../java/module 2/module-info.yaml | 7 + .../module 2/unit-challenge/description.md | 3 + .../java/module 2/unit-challenge/hint1.md | 3 + .../java/module 2/unit-challenge/hint2.md | 3 + .../module 2/unit-challenge/unit-info.yaml | 5 + .../java/module 2/unit-example/unit-info.yaml | 3 + .../learning-content/python/content-info.yaml | 3 + .../python/module 1/group/group-info.yaml | 7 + .../group/unit-challenge/description.md | 3 + .../module 1/group/unit-challenge/hint1.md | 3 + .../module 1/group/unit-challenge/hint2.md | 3 + .../group/unit-challenge/unit-info.yaml | 5 + .../group/unit-example/unit-info.yaml | 3 + .../python/module 1/intro-unit/unit-info.yaml | 3 + .../python/module 1/module-info.yaml | 7 + .../java/content-info.yaml | 2 + .../python/content-info.yaml | 2 + .../terraform-v2/modules/buckets/variables.tf | 2 +- .../modules/cloud_functions/main.tf | 13 +- .../terraform-v2/modules/iam/main.tf | 3 +- 56 files changed, 2613 insertions(+), 3 deletions(-) create mode 100644 learning/tour-of-beam/backend_v2/README.md create mode 100644 learning/tour-of-beam/backend_v2/cmd/ci_cd/ci_cd.go create mode 100644 learning/tour-of-beam/backend_v2/cmd/main.go create mode 100644 learning/tour-of-beam/backend_v2/datadir/WEB-INF/appengine-generated/local_db.bin create mode 100644 learning/tour-of-beam/backend_v2/datadir/WEB-INF/index.yaml create mode 100644 learning/tour-of-beam/backend_v2/datadir/env.yaml create mode 100644 learning/tour-of-beam/backend_v2/docker-compose.yml create mode 100644 learning/tour-of-beam/backend_v2/function.go create mode 100644 learning/tour-of-beam/backend_v2/go.mod create mode 100644 learning/tour-of-beam/backend_v2/go.sum create mode 100644 learning/tour-of-beam/backend_v2/internal/entity.go create mode 100644 learning/tour-of-beam/backend_v2/internal/fs_content/builders.go create mode 100644 learning/tour-of-beam/backend_v2/internal/fs_content/load.go create mode 100644 learning/tour-of-beam/backend_v2/internal/fs_content/load_test.go create mode 100644 learning/tour-of-beam/backend_v2/internal/fs_content/yaml.go create mode 100644 learning/tour-of-beam/backend_v2/internal/json.go create mode 100644 learning/tour-of-beam/backend_v2/internal/sdk.go create mode 100644 learning/tour-of-beam/backend_v2/internal/sdk_test.go create mode 100644 learning/tour-of-beam/backend_v2/internal/service/content.go create mode 100644 learning/tour-of-beam/backend_v2/internal/service/mock.go create mode 100644 learning/tour-of-beam/backend_v2/internal/storage/adapter.go create mode 100644 learning/tour-of-beam/backend_v2/internal/storage/datastore.go create mode 100644 learning/tour-of-beam/backend_v2/internal/storage/iface.go create mode 100644 learning/tour-of-beam/backend_v2/internal/storage/image/Dockerfile create mode 100644 learning/tour-of-beam/backend_v2/internal/storage/image/start-datastore.sh create mode 100644 learning/tour-of-beam/backend_v2/internal/storage/index.yaml create mode 100644 learning/tour-of-beam/backend_v2/internal/storage/schema.go create mode 100644 learning/tour-of-beam/backend_v2/samples/api/get_content_tree.json create mode 100644 learning/tour-of-beam/backend_v2/samples/api/get_unit_content.json create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/content-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/module-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/description.md create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint1.md create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint2.md create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-example/unit-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/module-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/description.md create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint1.md create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint2.md create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-example/unit-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/content-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/group-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/description.md create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint1.md create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint2.md create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/intro-unit/unit-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/module-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content_empty/java/content-info.yaml create mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content_empty/python/content-info.yaml diff --git a/learning/tour-of-beam/backend_v2/README.md b/learning/tour-of-beam/backend_v2/README.md new file mode 100644 index 000000000000..2be311203b2c --- /dev/null +++ b/learning/tour-of-beam/backend_v2/README.md @@ -0,0 +1,56 @@ + + +## Tour Of Beam Backend + +Backend provides the learning content tree for a given SDK, +and currently logged-in user's snippets and progress. +Currently it supports Java, Python, and Go Beam SDK. + +It is comprised of several Cloud Functions, with Firerstore in Datastore mode as a storage. +* list-sdks +* get-content-tree?sdk=(Java|Go|Python) +* get-unit-content?unitId= +TODO: add response schemas +TODO: add save functions info +TODO: add user token info + +### Datastore schema + +The storage is shared with Beam Playground backend, so that Tour Of Beam could access its entities in +pg_examples and pg_snippets. + +Conceptually, the learning tree is a tree with a root node in tb_learning_path, +having several children of tb_learning_module, and each module has its descendant nodes. +Node is either a group or a unit. + +Every module or unit has SDK-wide unique ID, which is provided by a content maintainer. +User's progress on a unit is tied to its ID, and if ID changes, the progress is lost. + +__Kinds__ +- tb_learning_path + + key: `(SDK_JAVA|SDK_PYTHON|SDK_GO)` + +- tb_learning_module + + key: `_` + + parentKey: Learning Path key SDK + +- tb_learning_node + + key: `_` + + parentKey: parent module/group key diff --git a/learning/tour-of-beam/backend_v2/cmd/ci_cd/ci_cd.go b/learning/tour-of-beam/backend_v2/cmd/ci_cd/ci_cd.go new file mode 100644 index 000000000000..9e7e109fecf0 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/cmd/ci_cd/ci_cd.go @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + "os" + + "beam.apache.org/learning/tour-of-beam/backend/internal/fs_content" + "beam.apache.org/learning/tour-of-beam/backend/internal/storage" + "cloud.google.com/go/datastore" +) + +var ( + repo storage.Iface + ctx context.Context +) + +func init() { + ctx = context.Background() + client, err := datastore.NewClient(ctx, "") + if err != nil { + log.Fatalf("new datastore client: %v", err) + } + repo = &storage.DatastoreDb{Client: client} +} + +func main() { + learningRoot := os.Getenv("TOB_LEARNING_ROOT") + log.Printf("Parsing learning-content at %q\n", learningRoot) + trees, err := fs_content.CollectLearningTree(learningRoot) + if err != nil { + log.Fatal(err) + } + + log.Printf("collected %v sdks\n", len(trees)) + if err = repo.SaveContentTrees(ctx, trees); err != nil { + log.Fatal(err) + } +} diff --git a/learning/tour-of-beam/backend_v2/cmd/main.go b/learning/tour-of-beam/backend_v2/cmd/main.go new file mode 100644 index 000000000000..b2b742534c68 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/cmd/main.go @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "log" + "os" + + // Blank-import the function package so the init() runs. + _ "beam.apache.org/learning/tour-of-beam/backend" + "github.com/GoogleCloudPlatform/functions-framework-go/funcframework" +) + +func main() { + // Use PORT environment variable, or default to 8080. + port := "8080" + if envPort := os.Getenv("PORT"); envPort != "" { + port = envPort + } + if err := funcframework.Start(port); err != nil { + log.Fatalf("funcframework.Start: %v\n", err) + } +} diff --git a/learning/tour-of-beam/backend_v2/datadir/WEB-INF/appengine-generated/local_db.bin b/learning/tour-of-beam/backend_v2/datadir/WEB-INF/appengine-generated/local_db.bin new file mode 100644 index 0000000000000000000000000000000000000000..ec17372c131cdf13aae1b935fa7b5b1f1bc60c00 GIT binary patch literal 6927 zcmds*-EZ4e6u__DWKGj_OVY1#2b0xpf>m0o_JwwPV4r1O)2&jwNx*~WCU=RANgDdGiAafK z-{b3h&iVPAd(N$2L3DNZKh@9Qp!(swqd~v{MY6aR5n4nI-*KiVdC71LMJ>@ z*6D*%^#`4nj3T`$+WXU;Z$~bEPbZxPaTQDXSt(~?G6(w1B0)wGtNF72%u#o>iPacA zCh3Qe z&dMrUGOd;U&YwksFVVx-)1zPh^26Y59&lOEOVvS|X=83n(6ejnsChw08fuGiva$GvUwFa)VRq zwkM_jG^4aGnZ>c#(#d)8%&`l{68-6P&?F_Cw^THi^v1>Ng-I&M%;K1CEagj~vP6A{ zCKDfV@kLdt6bQDoJQ;vdO!9~bV4RNu=Tt>kkXk!Uz=&DVk$c9GH00$x{U3)hZ#MD- zjA&BHkr;%jQ`Kh(=QY_B&eF?dG?s+@qG;qK(?moRsayo|t3MM6)1v4ax+2!h=#KI+ zo*aN9{#eyXqDj5vj`zjx-u>kEclXy=v8s#0#O!yu*dUBp&XU0Rbq>DDDG>z_V27w< zw^zp=g#DDq9FcYB^``Uj5ZJ?TfW9X$E@`r^BXiENkaPJlm@=?R-)yWk3|-^EHEwT% zyBeHsIZW-Ou)MFl!N7vs-?tH127Op^MTy2Rth=ks6pyc{l2H5F2Vt}* z(}*c><9p%b^q$l7Oqh(vC0$o5LIqpIKrhPlDArP86NpUU>yTLoYZmQ?eg1TvQtS4S_+BuB-r(SC9*JSgegsqK zZRHCFix0UlF5-?y7oS@`eeP_FEtl)_szm2aS53~@X6fO3v*G#!ML$$EhBIbE#u0Wy z7Mhc~33aU*>glix`)2La_3XktF}r16{eNaqJJdC9FcUh#=WqR%l+PG$KX^2_b+?H% zyuCW0i>jg#;*f24r_dL^Ppn*RvrpV?o?BKGT~`B}Rflv37x$I&76aGiPFRLl78{6( zY#|zqY&;2MmgOQ6o^_%q8TQKznkVa*nMB)EVT7|Xb8j12nV=;_)J+7mG#rCMt^N#G zr#m7z_w9g@Gqj`G zqVXd>)y8{lN2j^*^U9rdHGVv?b+=f+1Lbf~^N!|abx-@c$KZ~|idn>33+)TPx6yu+ z6Vx@XqZ&?=xw)~}f|DMP6YhECV$fQ%VF9Px=H*mF1WDEEp+4AK?QTwy5#P+T_q5>l zK!xR1Om@>w6m~kBW!bFI_)49yaO&jT`31JM$+hswkMFecNhcz^Y(=h{Nbcws*toH> zXtMESsEyBQw2Rwd<6am&er##(1Z(1Uk@=xna#qO)X+0B0>wl~K*eI}G6Ln$rZ7a^! Q6D!J2Kf15%9(v0E0^^%w!T "" { + svc = &service.Mock{} + } else { + // consumes DATASTORE_* env variables + client, err := datastore.NewClient(context.Background(), "") + if err != nil { + log.Fatalf("new datastore client: %v", err) + } + svc = &service.Svc{Repo: &storage.DatastoreDb{Client: client}} + } + + addHeader := AddHeader("Content-Type", "application/json") + ensureGet := EnsureMethod(http.MethodGet) + + // functions framework + functions.HTTP("sdkList", ensureGet(addHeader(sdkList))) + functions.HTTP("getContentTree", ensureGet(addHeader(ParseSdkParam(getContentTree)))) + functions.HTTP("getUnitContent", ensureGet(addHeader(ParseSdkParam(getUnitContent)))) +} + +// Get list of SDK names +// Used in both representation and accessing content. +func sdkList(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"names": ["Java", "Python", "Go"]}`) +} + +// Get the content tree for a given SDK and user +// Merges info from the default tree and per-user information: +// user code snippets and progress +// Required to be wrapped into ParseSdkParam middleware. +func getContentTree(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { + tree, err := svc.GetContentTree(r.Context(), sdk, nil /*TODO userId*/) + if err != nil { + log.Println("Get content tree error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") + return + } + + err = json.NewEncoder(w).Encode(tree) + if err != nil { + log.Println("Format content tree error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format content tree") + return + } +} + +// Get unit content +// Everything needed to render a learning unit: +// description, hints, code snippets +// Required to be wrapped into ParseSdkParam middleware. +func getUnitContent(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { + unitId := r.URL.Query().Get("unitId") + + unit, err := svc.GetUnitContent(r.Context(), sdk, unitId, nil /*TODO userId*/) + if err != nil { + log.Println("Get unit content error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") + return + } + if unit == nil { + log.Println("Get unit content error:", err) + finalizeErrResponse(w, http.StatusNotFound, NOT_FOUND, "unit not found") + return + } + + err = json.NewEncoder(w).Encode(unit) + if err != nil { + log.Println("Format unit content error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format unit content") + return + } +} diff --git a/learning/tour-of-beam/backend_v2/go.mod b/learning/tour-of-beam/backend_v2/go.mod new file mode 100644 index 000000000000..6601abeee276 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/go.mod @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module beam.apache.org/learning/tour-of-beam/backend + +go 1.16 + +require ( + github.com/GoogleCloudPlatform/functions-framework-go v1.5.3 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + cloud.google.com/go/datastore v1.8.0 + github.com/stretchr/testify v1.8.0 +) diff --git a/learning/tour-of-beam/backend_v2/go.sum b/learning/tour-of-beam/backend_v2/go.sum new file mode 100644 index 000000000000..45dd6a5e1909 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/go.sum @@ -0,0 +1,688 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.1 h1:vpK6iQWv/2uUeFJth4/cBHsQAGjn1iIE6AAlxipRaA0= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.8.0 h1:2qo2G7hABSeqswa+5Ga3+QB8/ZwKOJmDsCISM9scmsU= +cloud.google.com/go/datastore v1.8.0/go.mod h1:q1CpHVByTlXppdqTcu4LIhCsTn3fhtZ5R7+TajciO+M= +cloud.google.com/go/functions v1.0.0 h1:cOFEDJ3sgAFRjRULSUJ0Q8cw9qFa5JdpXIBWoNX5uDw= +cloud.google.com/go/functions v1.0.0/go.mod h1:O9KS8UweFVo6GbbbCBKh5yEzbW08PVkg2spe3RfPMd4= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/GoogleCloudPlatform/functions-framework-go v1.5.3 h1:Xx8uWT4hjgbjuXexbpU6V0yawWOdrbcAzZVyMYJvX8Q= +github.com/GoogleCloudPlatform/functions-framework-go v1.5.3/go.mod h1:pq+lZy4vONJ5fjd3q/B6QzWhfHPAbuVweLpxZzMOb9Y= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudevents/sdk-go/v2 v2.6.1 h1:yHtzgmeBvc0TZx1nrnvYXov1CSvkQyvhEhNMs8Z5Mmk= +github.com/cloudevents/sdk-go/v2 v2.6.1/go.mod h1:nlXhgFkf0uTopxmRXalyMwS2LG70cRGPrxzmjJgSG0U= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa h1:7MYGT2XEMam7Mtzv1yDUYXANedWvwk3HKkR3MyGowy8= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d h1:Zu/JngovGLVi6t2J3nmAf3AoTDwuzw85YZ3b9o4yU7s= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.84.0 h1:NMB9J4cCxs9xEm+1Z9QiO3eFvn7EnQj3Eo3hN6ugVlg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad h1:kqrS+lhvaMHCxul6sKQvKJ8nAAhlVItmZV822hYFH/U= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/learning/tour-of-beam/backend_v2/internal/entity.go b/learning/tour-of-beam/backend_v2/internal/entity.go new file mode 100644 index 000000000000..538715303452 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/entity.go @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +type Unit struct { + Id string `json:"unitId"` + Name string `json:"name"` + + // optional + Description string `json:"description,omitempty"` + Hints []string `json:"hints,omitempty"` + TaskSnippetId string `json:"taskSnippetId,omitempty"` + SolutionSnippetId string `json:"solutionSnippetId,omitempty"` + TaskName string `json:"-"` + SolutionName string `json:"-"` + + // optional, user-specific + UserSnippetId string `json:"userSnippetId,omitempty"` + IsCompleted string `json:"isCompleted,omitempty"` +} + +type NodeType int + +const ( + NODE_UNDEFINED NodeType = iota + NODE_UNIT + NODE_GROUP +) + +type Group struct { + Name string `json:"name"` + Nodes []Node `json:"nodes"` +} + +type Node struct { + Type NodeType `json:"type"` + Group *Group `json:"group,omitempty"` + Unit *Unit `json:"unit,omitempty"` +} + +type Module struct { + Id string `json:"moduleId"` + Name string `json:"name"` + Complexity string `json:"complexity"` + Nodes []Node `json:"nodes"` +} + +type ContentTree struct { + Sdk Sdk `json:"sdk"` + Modules []Module `json:"modules"` +} + +type CodeMessage struct { + Code string `json:"code"` + Message string `json:"message,omitempty"` +} diff --git a/learning/tour-of-beam/backend_v2/internal/fs_content/builders.go b/learning/tour-of-beam/backend_v2/internal/fs_content/builders.go new file mode 100644 index 000000000000..84895431cb08 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/fs_content/builders.go @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_content + +import ( + tob "beam.apache.org/learning/tour-of-beam/backend/internal" +) + +type UnitBuilder struct { + tob.Unit +} + +func NewUnitBuilder(info learningUnitInfo) UnitBuilder { + return UnitBuilder{tob.Unit{ + Id: info.Id, + Name: info.Name, + TaskName: info.TaskName, + SolutionName: info.SolutionName, + }} +} + +func (b *UnitBuilder) AddDescription(d string) { + b.Description = d +} + +func (b *UnitBuilder) AddHint(h string) { + b.Hints = append(b.Hints, h) +} + +func (b *UnitBuilder) Build() *tob.Unit { + return &b.Unit +} diff --git a/learning/tour-of-beam/backend_v2/internal/fs_content/load.go b/learning/tour-of-beam/backend_v2/internal/fs_content/load.go new file mode 100644 index 000000000000..8c9ea4271045 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/fs_content/load.go @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_content + +import ( + "fmt" + "io/fs" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + + tob "beam.apache.org/learning/tour-of-beam/backend/internal" +) + +const ( + contentInfoYaml = "content-info.yaml" + moduleInfoYaml = "module-info.yaml" + groupInfoYaml = "group-info.yaml" + unitInfoYaml = "unit-info.yaml" + + descriptionMd = "description.md" + hintMdRegexp = "hint[0-9]*.md" +) + +type learningPathInfo struct { + Sdk string + Content []string `yaml:"content"` +} + +type learningModuleInfo struct { + Id string + Name string + Complexity string + Content []string `yaml:"content"` +} + +type learningGroupInfo struct { + Name string + Content []string `yaml:"content"` +} + +type learningUnitInfo struct { + Id string + Name string + TaskName string + SolutionName string +} + +// Watch for duplicate ids. Not thread-safe! +type idsWatcher struct { + ids map[string]struct{} +} + +func (w *idsWatcher) CheckId(id string) { + if _, exists := w.ids[id]; exists { + log.Fatalf("Duplicate id: %v", id) + } + w.ids[id] = struct{}{} +} + +func NewIdsWatcher() idsWatcher { + return idsWatcher{make(map[string]struct{})} +} + +func collectUnit(infopath string, ids_watcher *idsWatcher) (unit *tob.Unit, err error) { + info := loadLearningUnitInfo(infopath) + log.Printf("Found Unit %v metadata at %v\n", info.Id, infopath) + ids_watcher.CheckId(info.Id) + builder := NewUnitBuilder(info) + + rootpath := filepath.Join(infopath, "..") + err = filepath.WalkDir(rootpath, + func(path string, d fs.DirEntry, err error) error { + switch { + // skip nested dirs + case path > rootpath && d.IsDir(): + return filepath.SkipDir + + case d.Name() == descriptionMd: + content, err := ioutil.ReadFile(path) + if err != nil { + return err + } + builder.AddDescription(string(content)) + + // Here we rely on that WalkDir entries are lexically sorted + case regexp.MustCompile(hintMdRegexp).MatchString(d.Name()): + content, err := ioutil.ReadFile(path) + if err != nil { + return err + } + builder.AddHint(string(content)) + } + return nil + }) + + return builder.Build(), err +} + +func collectGroup(infopath string, ids_watcher *idsWatcher) (*tob.Group, error) { + info := loadLearningGroupInfo(infopath) + log.Printf("Found Group %v metadata at %v\n", info.Name, infopath) + group := tob.Group{Name: info.Name} + for _, item := range info.Content { + node, err := collectNode(filepath.Join(infopath, "..", item), ids_watcher) + if err != nil { + return &group, err + } + group.Nodes = append(group.Nodes, node) + } + + return &group, nil +} + +// Collect node which is either a unit or a group. +func collectNode(rootpath string, ids_watcher *idsWatcher) (node tob.Node, err error) { + files, err := os.ReadDir(rootpath) + if err != nil { + return node, err + } + for _, f := range files { + switch f.Name() { + case unitInfoYaml: + node.Type = tob.NODE_UNIT + node.Unit, err = collectUnit(filepath.Join(rootpath, unitInfoYaml), ids_watcher) + case groupInfoYaml: + node.Type = tob.NODE_GROUP + node.Group, err = collectGroup(filepath.Join(rootpath, groupInfoYaml), ids_watcher) + } + } + if node.Type == tob.NODE_UNDEFINED { + return node, fmt.Errorf("node undefined at %v", rootpath) + } + return node, err +} + +func collectModule(infopath string, ids_watcher *idsWatcher) (tob.Module, error) { + info := loadLearningModuleInfo(infopath) + log.Printf("Found Module %v metadata at %v\n", info.Id, infopath) + ids_watcher.CheckId(info.Id) + module := tob.Module{Id: info.Id, Name: info.Name, Complexity: info.Complexity} + for _, item := range info.Content { + node, err := collectNode(filepath.Join(infopath, "..", item), ids_watcher) + if err != nil { + return tob.Module{}, err + } + module.Nodes = append(module.Nodes, node) + } + + return module, nil +} + +func collectSdk(infopath string) (tree tob.ContentTree, err error) { + ids_watcher := NewIdsWatcher() + + info := loadLearningPathInfo(infopath) + tree.Sdk = tob.ParseSdk(info.Sdk) + if tree.Sdk == tob.SDK_UNDEFINED { + return tree, fmt.Errorf("unknown SDK at %v", infopath) + } + log.Printf("Found Sdk %v metadata at %v\n", info.Sdk, infopath) + for _, item := range info.Content { + mod, err := collectModule(filepath.Join(infopath, "..", item, moduleInfoYaml), &ids_watcher) + if err != nil { + return tree, err + } + tree.Modules = append(tree.Modules, mod) + } + + return tree, nil +} + +// Build a content tree for each SDK +// Walk recursively through the learning-content dir, search for metadata files: +// content-info.yaml, module-info.yaml, unit-info.yaml. +func CollectLearningTree(rootpath string) (trees []tob.ContentTree, err error) { + err = filepath.WalkDir(rootpath, func(path string, d fs.DirEntry, err error) error { + // terminate walk on any error + if err != nil { + return err + } + if d.Name() == contentInfoYaml { + tree, err := collectSdk(path) + if err != nil { + return err + } + trees = append(trees, tree) + // don't walk into SDK subtree (already done by collectSdk) + return filepath.SkipDir + } + return nil + }) + + return trees, err +} diff --git a/learning/tour-of-beam/backend_v2/internal/fs_content/load_test.go b/learning/tour-of-beam/backend_v2/internal/fs_content/load_test.go new file mode 100644 index 000000000000..def3e9c1cd63 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/fs_content/load_test.go @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_content + +import ( + "path/filepath" + "testing" + + tob "beam.apache.org/learning/tour-of-beam/backend/internal" + "github.com/stretchr/testify/assert" +) + +func genUnitNode(id string) tob.Node { + return tob.Node{Type: tob.NODE_UNIT, Unit: &tob.Unit{ + Id: id, Name: "Challenge Name", + Description: "## Challenge description\n\nbla bla bla", + Hints: []string{ + "## Hint 1\n\napply yourself :)", + "## Hint 2\n\napply more", + }, + }} +} + +func TestSample(t *testing.T) { + trees, err := CollectLearningTree(filepath.Join("..", "..", "samples", "learning-content")) + assert.Nil(t, err) + assert.Equal(t, 2, len(trees)) + assert.Equal(t, tob.ContentTree{ + Sdk: tob.SDK_JAVA, + Modules: []tob.Module{ + { + Id: "module1", Name: "Module One", Complexity: "BASIC", + Nodes: []tob.Node{ + {Type: tob.NODE_UNIT, Unit: &tob.Unit{Id: "example1", Name: "Example Unit Name"}}, + genUnitNode("challenge1"), + }, + }, + { + Id: "module2", Name: "Module Two", Complexity: "MEDIUM", + Nodes: []tob.Node{ + {Type: tob.NODE_UNIT, Unit: &tob.Unit{Id: "example21", Name: "Example Unit Name"}}, + genUnitNode("challenge21"), + }, + }, + }, + }, trees[0]) + assert.Equal(t, tob.ContentTree{ + Sdk: tob.SDK_PYTHON, + Modules: []tob.Module{ + { + Id: "module1", Name: "Module One", Complexity: "BASIC", + Nodes: []tob.Node{ + {Type: tob.NODE_UNIT, Unit: &tob.Unit{Id: "intro-unit", Name: "Intro Unit Name"}}, + { + Type: tob.NODE_GROUP, Group: &tob.Group{ + Name: "The Group", Nodes: []tob.Node{ + {Type: tob.NODE_UNIT, Unit: &tob.Unit{Id: "example1", Name: "Example Unit Name"}}, + genUnitNode("challenge1"), + }, + }, + }, + }, + }, + }, + }, trees[1]) +} diff --git a/learning/tour-of-beam/backend_v2/internal/fs_content/yaml.go b/learning/tour-of-beam/backend_v2/internal/fs_content/yaml.go new file mode 100644 index 000000000000..0546637a0d55 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/fs_content/yaml.go @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_content + +import ( + "io/ioutil" + "log" + + "gopkg.in/yaml.v3" +) + +// Could have done it in generics if 1.18 was supported in GCF +// Fatals on error. +func loadLearningPathInfo(path string) (info learningPathInfo) { + buf, err := ioutil.ReadFile(path) + if err != nil { + log.Fatal(err) + } + + err = yaml.Unmarshal(buf, &info) + if err != nil { + log.Fatal(err) + } + + return info +} + +func loadLearningModuleInfo(path string) (info learningModuleInfo) { + buf, err := ioutil.ReadFile(path) + if err != nil { + log.Fatal(err) + } + + err = yaml.Unmarshal(buf, &info) + if err != nil { + log.Fatal(err) + } + + return info +} + +func loadLearningGroupInfo(path string) (info learningGroupInfo) { + buf, err := ioutil.ReadFile(path) + if err != nil { + log.Fatal(err) + } + + err = yaml.Unmarshal(buf, &info) + if err != nil { + log.Fatal(err) + } + + return info +} + +func loadLearningUnitInfo(path string) (info learningUnitInfo) { + buf, err := ioutil.ReadFile(path) + if err != nil { + log.Fatal(err) + } + + err = yaml.Unmarshal(buf, &info) + if err != nil { + log.Fatal(err) + } + + return info +} diff --git a/learning/tour-of-beam/backend_v2/internal/json.go b/learning/tour-of-beam/backend_v2/internal/json.go new file mode 100644 index 000000000000..793f04b213c8 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/json.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +func (nt NodeType) MarshalText() ([]byte, error) { + var typ string + switch nt { + case NODE_UNIT: + typ = "unit" + case NODE_GROUP: + typ = "group" + default: + panic("NodeType not defined") + } + return []byte(typ), nil +} diff --git a/learning/tour-of-beam/backend_v2/internal/sdk.go b/learning/tour-of-beam/backend_v2/internal/sdk.go new file mode 100644 index 000000000000..a1451d183755 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/sdk.go @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +type Sdk string + +const ( + SDK_UNDEFINED Sdk = "" + SDK_GO Sdk = "Go" + SDK_PYTHON Sdk = "Python" + SDK_JAVA Sdk = "Java" + SDK_SCIO Sdk = "SCIO" +) + +func (s Sdk) String() string { + return string(s) +} + +// Parse sdk from string names, f.e. "Java" -> Sdk.GO_JAVA +// Returns SDK_UNDEFINED on error. +func ParseSdk(s string) Sdk { + switch s { + case "Go": + return SDK_GO + case "Python": + return SDK_PYTHON + case "Java": + return SDK_JAVA + case "SCIO": + return SDK_SCIO + default: + return SDK_UNDEFINED + } +} + +func SdksList() [4]string { + return [4]string{"Java", "Python", "Go", "SCIO"} +} diff --git a/learning/tour-of-beam/backend_v2/internal/sdk_test.go b/learning/tour-of-beam/backend_v2/internal/sdk_test.go new file mode 100644 index 000000000000..562679952c1a --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/sdk_test.go @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import "testing" + +func TestParse(t *testing.T) { + for _, s := range []struct { + str string + expected Sdk + }{ + {"Go", SDK_GO}, + {"Python", SDK_PYTHON}, + {"Java", SDK_JAVA}, + {"SCIO", SDK_SCIO}, + {"Bad", SDK_UNDEFINED}, + {"", SDK_UNDEFINED}, + } { + if parsed := ParseSdk(s.str); parsed != s.expected { + t.Errorf("Failed to parse %v: got %v (expected %v)", s.str, parsed, s.expected) + } + } +} + +func TestSerialize(t *testing.T) { + for _, s := range []struct { + expected string + sdk Sdk + }{ + {"Go", SDK_GO}, + {"Python", SDK_PYTHON}, + {"Java", SDK_JAVA}, + {"SCIO", SDK_SCIO}, + {"", SDK_UNDEFINED}, + } { + if txt := s.sdk.String(); txt != s.expected { + t.Errorf("Failed to serialize %v to string: got %v (expected %v)", s.sdk, txt, s.expected) + } + } +} + +func TestSdkList(t *testing.T) { + if SdksList() != [4]string{"Java", "Python", "Go", "SCIO"} { + t.Error("Sdk list mismatch: ", SdksList()) + } +} diff --git a/learning/tour-of-beam/backend_v2/internal/service/content.go b/learning/tour-of-beam/backend_v2/internal/service/content.go new file mode 100644 index 000000000000..2644e5a1a143 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/service/content.go @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "context" + + tob "beam.apache.org/learning/tour-of-beam/backend/internal" + "beam.apache.org/learning/tour-of-beam/backend/internal/storage" +) + +type IContent interface { + GetContentTree(ctx context.Context, sdk tob.Sdk, userId *string) (tob.ContentTree, error) + GetUnitContent(ctx context.Context, sdk tob.Sdk, unitId string, userId *string) (*tob.Unit, error) +} + +type Svc struct { + Repo storage.Iface +} + +func (s *Svc) GetContentTree(ctx context.Context, sdk tob.Sdk, userId *string) (ct tob.ContentTree, err error) { + // TODO enrich tree with user-specific state (isCompleted) + return s.Repo.GetContentTree(ctx, sdk) +} + +func (s *Svc) GetUnitContent(ctx context.Context, sdk tob.Sdk, unitId string, userId *string) (unit *tob.Unit, err error) { + // TODO enrich unit with user-specific state: isCompleted, userSnippetId + return s.Repo.GetUnitContent(ctx, sdk, unitId) +} diff --git a/learning/tour-of-beam/backend_v2/internal/service/mock.go b/learning/tour-of-beam/backend_v2/internal/service/mock.go new file mode 100644 index 000000000000..eee1c292b516 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/service/mock.go @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "context" + "encoding/json" + "io/ioutil" + "path" + "runtime" + + tob "beam.apache.org/learning/tour-of-beam/backend/internal" +) + +func getSamplesPath() string { + _, filepath, _, _ := runtime.Caller(1) + return path.Join(path.Dir(filepath), "..", "..", "samples", "api") +} + +type Mock struct{} + +// check if the interface is implemented. +var _ IContent = &Mock{} + +func (d *Mock) GetContentTree(_ context.Context, sdk tob.Sdk, userId *string) (ct tob.ContentTree, err error) { + content, _ := ioutil.ReadFile(path.Join(getSamplesPath(), "content_tree.json")) + _ = json.Unmarshal(content, &ct) + return ct, nil +} + +func (d *Mock) GetUnitContent(_ context.Context, sdk tob.Sdk, unitId string, userId *string) (u *tob.Unit, err error) { + content, _ := ioutil.ReadFile(path.Join(getSamplesPath(), "unit.json")) + err = json.Unmarshal(content, &u) + return u, err +} diff --git a/learning/tour-of-beam/backend_v2/internal/storage/adapter.go b/learning/tour-of-beam/backend_v2/internal/storage/adapter.go new file mode 100644 index 000000000000..ac7a1c2b25f3 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/storage/adapter.go @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + + tob "beam.apache.org/learning/tour-of-beam/backend/internal" + "cloud.google.com/go/datastore" +) + +func sdkToKey(sdk tob.Sdk) string { + switch sdk { + case tob.SDK_GO: + return "SDK_GO" + case tob.SDK_PYTHON: + return "SDK_PYTHON" + case tob.SDK_JAVA: + return "SDK_JAVA" + case tob.SDK_SCIO: + return "SDK_SCIO" + } + panic(fmt.Sprintf("Undefined key for sdk: %s", sdk)) +} + +func pgNameKey(kind, nameId string, parentKey *datastore.Key) (key *datastore.Key) { + key = datastore.NameKey(kind, nameId, parentKey) + key.Namespace = PgNamespace + return key +} + +// Get entity key from sdk & entity ID +// SDK_JAVA_{entityID}. +func datastoreKey(kind string, sdk tob.Sdk, id string, parent *datastore.Key) *datastore.Key { + name := fmt.Sprintf("%s_%s", sdkToKey(sdk), id) + return pgNameKey(kind, name, parent) +} + +func MakeUnitNode(unit *tob.Unit, order, level int) *TbLearningNode { + if unit == nil { + return nil + } + return &TbLearningNode{ + Id: unit.Id, + Name: unit.Name, + + Type: tob.NODE_UNIT, + Order: order, + Level: level, + + Unit: &TbLearningUnit{ + Id: unit.Id, + Name: unit.Name, + + Description: unit.Description, + Hints: unit.Hints, + TaskSnippetId: unit.TaskSnippetId, + SolutionSnippetId: unit.SolutionSnippetId, + }, + } +} + +func MakeGroupNode(group *tob.Group, order, level int) *TbLearningNode { + if group == nil { + return nil + } + return &TbLearningNode{ + // ID doesn't make much sense for groups, + // but we have to define it to include in queries + Id: group.Name, + Name: group.Name, + + Type: tob.NODE_GROUP, + Order: order, + Level: level, + + Group: &TbLearningGroup{ + Name: group.Name, + }, + } +} + +// Depending on the projection, we either convert TbLearningUnit to a model +// Or we use common fields Id, Name to make it. +func FromDatastoreUnit(tbUnit *TbLearningUnit, id, name string) *tob.Unit { + if tbUnit == nil { + return &tob.Unit{Id: id, Name: name} + } + return &tob.Unit{ + Id: tbUnit.Id, + Name: tbUnit.Name, + Description: tbUnit.Description, + Hints: tbUnit.Hints, + TaskSnippetId: tbUnit.TaskSnippetId, + SolutionSnippetId: tbUnit.SolutionSnippetId, + } +} + +// Depending on the projection, we either convert TbLearningGroup to a model +// Or we use common field Name to make it. +func FromDatastoreGroup(tbGroup *TbLearningGroup, name string) *tob.Group { + if tbGroup == nil { + return &tob.Group{Name: name} + } + return &tob.Group{ + Name: tbGroup.Name, + } +} + +func FromDatastoreNode(tbNode TbLearningNode) tob.Node { + node := tob.Node{ + Type: tbNode.Type, + } + switch tbNode.Type { + case tob.NODE_GROUP: + node.Group = FromDatastoreGroup(tbNode.Group, tbNode.Name) + case tob.NODE_UNIT: + node.Unit = FromDatastoreUnit(tbNode.Unit, tbNode.Id, tbNode.Name) + default: + panic("undefined node type") + } + return node +} + +func MakeDatastoreModule(mod *tob.Module, order int) *TbLearningModule { + return &TbLearningModule{ + Id: mod.Id, + Name: mod.Name, + Complexity: mod.Complexity, + + Order: order, + } +} diff --git a/learning/tour-of-beam/backend_v2/internal/storage/datastore.go b/learning/tour-of-beam/backend_v2/internal/storage/datastore.go new file mode 100644 index 000000000000..62c55b2ba5b2 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/storage/datastore.go @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "fmt" + "log" + + tob "beam.apache.org/learning/tour-of-beam/backend/internal" + "cloud.google.com/go/datastore" +) + +type DatastoreDb struct { + Client *datastore.Client +} + +// Query modules structure and content (recursively). +func (d *DatastoreDb) collectModules(ctx context.Context, tx *datastore.Transaction, + rootKey *datastore.Key, +) ([]tob.Module, error) { + // Custom index.yaml should be applied for this query to work + // (Ancestor + Order) + modules := make([]tob.Module, 0) + var tbMods []TbLearningModule + queryModules := datastore.NewQuery(TbLearningModuleKind). + Namespace(PgNamespace). + Ancestor(rootKey). + Order("order"). + Transaction(tx) + _, err := d.Client.GetAll(ctx, queryModules, &tbMods) + if err != nil { + return modules, fmt.Errorf("error querying modules for %v: %w", rootKey, err) + } + + for _, tbMod := range tbMods { + mod := tob.Module{Id: tbMod.Id, Name: tbMod.Name, Complexity: tbMod.Complexity} + mod.Nodes, err = d.collectNodes(ctx, tx, tbMod.Key, 0) + if err != nil { + return modules, err + } + modules = append(modules, mod) + } + return modules, nil +} + +// Get a group recursively. +// Params: +// - parentKey +// - level: depth of a node's children +// Recursively query/collect for each subgroup key, with level = level + 1. +func (d *DatastoreDb) collectNodes(ctx context.Context, tx *datastore.Transaction, + parentKey *datastore.Key, level int, +) (nodes []tob.Node, err error) { + var tbNodes []TbLearningNode + + // Custom index.yaml should be applied for this query to work + queryNodes := datastore.NewQuery(TbLearningNodeKind). + Namespace(PgNamespace). + Ancestor(parentKey). + FilterField("level", "=", level). + Project("type", "id", "name"). + Order("order"). + Transaction(tx) + if _, err = d.Client.GetAll(ctx, queryNodes, &tbNodes); err != nil { + return nodes, fmt.Errorf("getting children of node %v: %w", parentKey, err) + } + + // traverse the nodes which are groups, with level=level+1 + nodes = make([]tob.Node, 0, len(tbNodes)) + for _, tbNode := range tbNodes { + node := FromDatastoreNode(tbNode) + if node.Type == tob.NODE_GROUP { + node.Group.Nodes, err = d.collectNodes(ctx, tx, tbNode.Key, level+1) + } + if err != nil { + return nodes, err + } + nodes = append(nodes, node) + } + + return nodes, nil +} + +// Get learning content tree for SDK. +func (d *DatastoreDb) GetContentTree(ctx context.Context, sdk tob.Sdk) (tree tob.ContentTree, err error) { + var tbLP TbLearningPath + tree.Sdk = sdk + + _, err = d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error { + rootKey := pgNameKey(TbLearningPathKind, sdkToKey(sdk), nil) + if err := d.Client.Get(ctx, rootKey, &tbLP); err != nil { + return fmt.Errorf("error querying learning_path: %w", err) + } + tree.Modules, err = d.collectModules(ctx, tx, rootKey) + if err != nil { + return err + } + return nil + }, datastore.ReadOnly) + + return tree, err +} + +// Helper to clear all ToB Datastore entities related to a particular SDK +// They have one common ancestor key in tb_learning_path. +func (d *DatastoreDb) clearContentTree(ctx context.Context, tx *datastore.Transaction, sdk tob.Sdk) error { + rootKey := pgNameKey(TbLearningPathKind, sdkToKey(sdk), nil) + q := datastore.NewQuery(""). + Namespace(PgNamespace). + Ancestor(rootKey). + KeysOnly(). + Transaction(tx) + keys, err := d.Client.GetAll(ctx, q, nil) + if err != nil { + return err + } + + for _, key := range keys { + log.Println("deleting ", key) + } + + err = tx.DeleteMulti(keys) + if err != nil { + return err + } + return tx.Delete(rootKey) +} + +// Serialize a content tree to Datastore. +func (d *DatastoreDb) saveContentTree(tx *datastore.Transaction, tree *tob.ContentTree) error { + sdk := tree.Sdk + + saveUnit := func(unit *tob.Unit, order, level int, parentKey *datastore.Key) error { + unitKey := datastoreKey(TbLearningNodeKind, tree.Sdk, unit.Id, parentKey) + _, err := tx.Put(unitKey, MakeUnitNode(unit, order, level)) + if err != nil { + return fmt.Errorf("failed to put unit: %w", err) + } + return nil + } + + // transaction-wide autoincremented Id + // could have used numericID keys, if there was no transaction: + // incomplete keys are resolved after Tx commit, and + // we need to reference them in child nodes + var groupId int = 0 + genGroupKey := func(parentKey *datastore.Key) *datastore.Key { + groupId++ + return datastoreKey(TbLearningNodeKind, + tree.Sdk, fmt.Sprintf("group%v", groupId), parentKey) + } + + var saveNode func(tob.Node, int, int, *datastore.Key) error + saveGroup := func(group *tob.Group, order, level int, parentKey *datastore.Key) error { + groupKey := genGroupKey(parentKey) + if _, err := tx.Put(groupKey, MakeGroupNode(group, order, level)); err != nil { + return fmt.Errorf("failed to put group: %w", err) + } + for order, node := range group.Nodes { + if err := saveNode(node, order, level+1, groupKey); err != nil { + return err + } + } + return nil + } + + saveNode = func(node tob.Node, order, level int, parentKey *datastore.Key) error { + if node.Type == tob.NODE_UNIT { + return saveUnit(node.Unit, order, level, parentKey) + } else if node.Type == tob.NODE_GROUP { + return saveGroup(node.Group, order, level, parentKey) + } + + return fmt.Errorf("unknown datastore node type: %v", node.Type) + } + + rootKey := pgNameKey(TbLearningPathKind, sdkToKey(tree.Sdk), nil) + tbLP := TbLearningPath{Name: tree.Sdk.String()} + if _, err := tx.Put(rootKey, &tbLP); err != nil { + return fmt.Errorf("failed to put learning_path: %w", err) + } + + for order, mod := range tree.Modules { + modKey := datastoreKey(TbLearningModuleKind, sdk, mod.Id, rootKey) + if _, err := tx.Put(modKey, MakeDatastoreModule(&mod, order)); err != nil { + return fmt.Errorf("failed to put module: %w", err) + } + for order, node := range mod.Nodes { + if err := saveNode(node, order /*module node is at level 0*/, 0, modKey); err != nil { + return err + } + } + } + + return nil +} + +// Re-create content trees for each SDK in separate transaction. +func (d *DatastoreDb) SaveContentTrees(ctx context.Context, trees []tob.ContentTree) error { + for _, tree := range trees { + log.Println("Saving sdk tree", tree.Sdk) + _, err := d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error { + if err := d.clearContentTree(ctx, tx, tree.Sdk); err != nil { + return err + } + return d.saveContentTree(tx, &tree) + }) + if err != nil { + return err + } + } + + return nil +} + +// Get learning unit content by unitId +func (d *DatastoreDb) GetUnitContent(ctx context.Context, sdk tob.Sdk, unitId string) (unit *tob.Unit, err error) { + var tbNodes []TbLearningNode + rootKey := pgNameKey(TbLearningPathKind, sdkToKey(sdk), nil) + + query := datastore.NewQuery(TbLearningNodeKind). + Namespace(PgNamespace). + Ancestor(rootKey). + FilterField("id", "=", unitId) + + _, err = d.Client.GetAll(ctx, query, &tbNodes) + if err != nil { + return nil, fmt.Errorf("query unit failed: %w", err) + } + + switch { + case len(tbNodes) == 0: + return nil, nil + case len(tbNodes) > 1: + return nil, fmt.Errorf("query by unitId returned %v units", len(tbNodes)) + } + + node := FromDatastoreNode(tbNodes[0]) + if node.Type != tob.NODE_UNIT { + return nil, fmt.Errorf("wrong node type: %v, unit expected", node.Type) + } + return node.Unit, nil +} + +// check if the interface is implemented. +var _ Iface = &DatastoreDb{} diff --git a/learning/tour-of-beam/backend_v2/internal/storage/iface.go b/learning/tour-of-beam/backend_v2/internal/storage/iface.go new file mode 100644 index 000000000000..f81a28e4ce52 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/storage/iface.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + tob "beam.apache.org/learning/tour-of-beam/backend/internal" +) + +type Iface interface { + GetContentTree(ctx context.Context, sdk tob.Sdk) (tob.ContentTree, error) + SaveContentTrees(ctx context.Context, trees []tob.ContentTree) error + + GetUnitContent(ctx context.Context, sdk tob.Sdk, unitId string) (*tob.Unit, error) +} diff --git a/learning/tour-of-beam/backend_v2/internal/storage/image/Dockerfile b/learning/tour-of-beam/backend_v2/internal/storage/image/Dockerfile new file mode 100644 index 000000000000..f8f5a4b92b8c --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/storage/image/Dockerfile @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Version. Can change in build progress +ARG GCLOUD_SDK_VERSION=397.0.0-emulators + +# Use google cloud sdk +FROM google/cloud-sdk:$GCLOUD_SDK_VERSION + +# Volume to persist Datastore data +VOLUME /opt/data + +# RUN mkdir -p /opt/data/WEB-INF +# COPY index.yaml /opt/data/WEB-INF/index.yaml +COPY start-datastore.sh . + +EXPOSE 8081 + +ENTRYPOINT ["./start-datastore.sh"] diff --git a/learning/tour-of-beam/backend_v2/internal/storage/image/start-datastore.sh b/learning/tour-of-beam/backend_v2/internal/storage/image/start-datastore.sh new file mode 100644 index 000000000000..bf0ccf3f7eba --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/storage/image/start-datastore.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Check user environment variable +if [[ -z "${DATASTORE_PROJECT_ID}" ]]; then + echo "Missing DATASTORE_PROJECT_ID environment variable" >&2 + exit 1 +fi + +if [[ -z "${DATASTORE_LISTEN_ADDRESS}" ]]; then + echo "Missing DATASTORE_LISTEN_ADDRESS environment variable" >&2 + exit 1 +fi + +options=${options:1} +# Check for datastore options +while [ ! $# -eq 0 ] +do + case "$1" in + --store-on-disk) + options="$options --store-on-disk" + shift + ;; + --no-store-on-disk) + options="$options --no-store-on-disk" + shift + ;; + --consistency=*) + consistency=${1#*=} + options="$options --consistency=$consistency" + shift + ;; + *) + echo "Invalid option: $1. Use: --store-on-disk, --no-store-on-disk, --consistency=[0.0-1.0]" + exit 1 + ;; + esac +done + +# Config gcloud project +gcloud config set project ${DATASTORE_PROJECT_ID} + +# Start emulator +gcloud beta emulators datastore start \ + --data-dir=/opt/data \ + --host-port=${DATASTORE_LISTEN_ADDRESS} \ + ${options} diff --git a/learning/tour-of-beam/backend_v2/internal/storage/index.yaml b/learning/tour-of-beam/backend_v2/internal/storage/index.yaml new file mode 100644 index 000000000000..fa78d72f9481 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/storage/index.yaml @@ -0,0 +1,20 @@ +indexes: +# AUTOGENERATED + +# This index.yaml is automatically updated whenever the Cloud Datastore +# emulator detects that a new type of query is run. If you want to manage the +# index.yaml file manually, remove the "# AUTOGENERATED" marker line above. +# If you want to manage some indexes manually, move them above the marker line. + +- kind: "tb_learning_module" + ancestor: yes + properties: + - name: "order" +- kind: "tb_learning_node" + ancestor: yes + properties: + - name: "level" + - name: "order" + - name: "id" + - name: "name" + - name: "type" diff --git a/learning/tour-of-beam/backend_v2/internal/storage/schema.go b/learning/tour-of-beam/backend_v2/internal/storage/schema.go new file mode 100644 index 000000000000..4e2b31679203 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/internal/storage/schema.go @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + tob "beam.apache.org/learning/tour-of-beam/backend/internal" + "cloud.google.com/go/datastore" +) + +// ToB Datastore schema +// - Content tree has a root entity in tb_learning_path, +// descendant entities in tb_learning_module/group/unit have it as a common ancestor +// - learning path consists of modules, modules consist of groups and units +// - Ordering is established by "order" property +// - To limit ancestor queries by only first-level descendants, "level" property is used + +const ( + PgNamespace = "Tour" + + TbLearningPathKind = "tb_learning_path" + TbLearningModuleKind = "tb_learning_module" + TbLearningNodeKind = "tb_learning_node" + + PgSnippetsKind = "pg_snippets" + PgSdksKind = "pg_sdks" +) + +// tb_learning_path. +type TbLearningPath struct { + Key *datastore.Key `datastore:"__key__"` + Name string `datastore:"name"` +} + +// tb_learning_module. +type TbLearningModule struct { + Key *datastore.Key `datastore:"__key__"` + Id string `datastore:"id"` + Name string `datastore:"name"` + Complexity string `datastore:"complexity"` + + // internal, only db + Order int `datastore:"order"` +} + +// tb_learning_node.group. +type TbLearningGroup struct { + Name string `datastore:"name"` +} + +// tb_learning_node.unit +// Learning Unit content. +type TbLearningUnit struct { + Id string `datastore:"id"` + Name string `datastore:"name"` + Description string `datastore:"description,noindex"` + Hints []string `datastore:"hints,noindex"` + + TaskSnippetId string `datastore:"taskSnippetId"` + SolutionSnippetId string `datastore:"solutionSnippetId"` +} + +// tb_learning_node +// Container for learning tree nodes, which are either groups or units. +type TbLearningNode struct { + Type tob.NodeType `datastore:"type"` + // common fields, duplicate same fields from the nested entities + // (needed to allow projection when getting the content tree) + Id string `datastore:"id"` + Name string `datastore:"name"` + + // type-specific nested info + Unit *TbLearningUnit `datastore:"unit,noindex"` + Group *TbLearningGroup `datastore:"group,noindex"` + + // internal datastore-only fields + // key: _ + Key *datastore.Key `datastore:"__key__"` + Order int `datastore:"order"` + Level int `datastore:"level"` +} + +type PgSnippets struct { + Key *datastore.Key `datastore:"__key__"` + Origin string `datastore:"origin"` + Sdk *datastore.Key `datastore:"sdk"` +} diff --git a/learning/tour-of-beam/backend_v2/samples/api/get_content_tree.json b/learning/tour-of-beam/backend_v2/samples/api/get_content_tree.json new file mode 100644 index 000000000000..cf8c40b6f73b --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/api/get_content_tree.json @@ -0,0 +1,41 @@ +{ + "modules" : [ + { + "complexity" : "BASIC", + "moduleId" : "module1", + "name" : "Module One", + "nodes" : [ + { + "type" : "unit", + "unit" : { + "name" : "Intro Unit Name", + "unitId" : "intro-unit" + } + }, + { + "group" : { + "name" : "The Group", + "nodes" : [ + { + "type" : "unit", + "unit" : { + "name" : "Example Unit Name", + "unitId" : "example1" + } + }, + { + "type" : "unit", + "unit" : { + "name" : "Challenge Name", + "unitId" : "challenge1" + } + } + ] + }, + "type" : "group" + } + ] + } + ], + "sdk" : "Python" +} \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/api/get_unit_content.json b/learning/tour-of-beam/backend_v2/samples/api/get_unit_content.json new file mode 100644 index 000000000000..a18305870ceb --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/api/get_unit_content.json @@ -0,0 +1,10 @@ +{ + "unitId": "1.1", + "name": "Basic concepts", + "description": "Lorem ipsum...", + "hint": "Try to use this method....", + "assignment": "assignmentSnippetId", + "solution": "solutionSnippetId", + "userSnippet": "userSnippetId", + "completed": true +} diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/content-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/content-info.yaml new file mode 100644 index 000000000000..0d6a17105315 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/content-info.yaml @@ -0,0 +1,4 @@ +sdk: Java +content: + - module 1 + - module 2 \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/module-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/module-info.yaml new file mode 100644 index 000000000000..04384cb56b31 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/module-info.yaml @@ -0,0 +1,7 @@ +id: module1 +name: Module One +complexity: BASIC +content: +- unit-example +- unit-challenge + diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/description.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/description.md new file mode 100644 index 000000000000..1906064ed20f --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/description.md @@ -0,0 +1,3 @@ +## Challenge description + +bla bla bla \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint1.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint1.md new file mode 100644 index 000000000000..e5d1a67fa906 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint1.md @@ -0,0 +1,3 @@ +## Hint 1 + +apply yourself :) \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint2.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint2.md new file mode 100644 index 000000000000..0c425751be9f --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint2.md @@ -0,0 +1,3 @@ +## Hint 2 + +apply more \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml new file mode 100644 index 000000000000..e20b873a3e05 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml @@ -0,0 +1,5 @@ +id: challenge1 +name: Challenge Name +complexity: BASIC +taskName: ChallengeTask +solutionName: ChallengeSolution \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-example/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-example/unit-info.yaml new file mode 100644 index 000000000000..ed0c81854cb0 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-example/unit-info.yaml @@ -0,0 +1,3 @@ +id: example1 +name: Example Unit Name +taskName: ExampleName \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/module-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/module-info.yaml new file mode 100644 index 000000000000..e152c45d22e1 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/module-info.yaml @@ -0,0 +1,7 @@ +id: module2 +name: Module Two +complexity: MEDIUM +content: +- unit-example +- unit-challenge + diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/description.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/description.md new file mode 100644 index 000000000000..1906064ed20f --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/description.md @@ -0,0 +1,3 @@ +## Challenge description + +bla bla bla \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint1.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint1.md new file mode 100644 index 000000000000..e5d1a67fa906 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint1.md @@ -0,0 +1,3 @@ +## Hint 1 + +apply yourself :) \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint2.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint2.md new file mode 100644 index 000000000000..0c425751be9f --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint2.md @@ -0,0 +1,3 @@ +## Hint 2 + +apply more \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml new file mode 100644 index 000000000000..ac65e8357d4c --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml @@ -0,0 +1,5 @@ +id: challenge21 +name: Challenge Name +complexity: BASIC +taskName: ChallengeTask +solutionName: ChallengeSolution \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-example/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-example/unit-info.yaml new file mode 100644 index 000000000000..674a6da71941 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-example/unit-info.yaml @@ -0,0 +1,3 @@ +id: example21 +name: Example Unit Name +taskName: ExampleName \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/content-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/content-info.yaml new file mode 100644 index 000000000000..ee9d329b5030 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/python/content-info.yaml @@ -0,0 +1,3 @@ +sdk: Python +content: + - module 1 diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/group-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/group-info.yaml new file mode 100644 index 000000000000..1619bc19fb91 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/group-info.yaml @@ -0,0 +1,7 @@ +id: group +name: The Group +complexity: BASIC +content: +- unit-example +- unit-challenge + diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/description.md b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/description.md new file mode 100644 index 000000000000..1906064ed20f --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/description.md @@ -0,0 +1,3 @@ +## Challenge description + +bla bla bla \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint1.md b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint1.md new file mode 100644 index 000000000000..e5d1a67fa906 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint1.md @@ -0,0 +1,3 @@ +## Hint 1 + +apply yourself :) \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint2.md b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint2.md new file mode 100644 index 000000000000..0c425751be9f --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint2.md @@ -0,0 +1,3 @@ +## Hint 2 + +apply more \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml new file mode 100644 index 000000000000..e20b873a3e05 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml @@ -0,0 +1,5 @@ +id: challenge1 +name: Challenge Name +complexity: BASIC +taskName: ChallengeTask +solutionName: ChallengeSolution \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml new file mode 100644 index 000000000000..ed0c81854cb0 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml @@ -0,0 +1,3 @@ +id: example1 +name: Example Unit Name +taskName: ExampleName \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/intro-unit/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/intro-unit/unit-info.yaml new file mode 100644 index 000000000000..1bcdb8d1abd1 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/intro-unit/unit-info.yaml @@ -0,0 +1,3 @@ +id: intro-unit +name: Intro Unit Name +taskName: IntroTask \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/module-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/module-info.yaml new file mode 100644 index 000000000000..024d12fdc69c --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/module-info.yaml @@ -0,0 +1,7 @@ +id: module1 +name: Module One +complexity: BASIC +content: +- intro-unit +- group + diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content_empty/java/content-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content_empty/java/content-info.yaml new file mode 100644 index 000000000000..66ecd69e089d --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content_empty/java/content-info.yaml @@ -0,0 +1,2 @@ +sdk: Java +content: [] \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content_empty/python/content-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content_empty/python/content-info.yaml new file mode 100644 index 000000000000..cf165934c106 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/samples/learning-content_empty/python/content-info.yaml @@ -0,0 +1,2 @@ +sdk: Python +content: [] diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf index 0416cc20cbfd..924de05f0e8c 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -14,6 +14,6 @@ variable "project_id" { data "archive_file" "source" { type = "zip" - source_dir = "../../../backend" + source_dir = "../../../backend_" output_path = "/tmp/backend.zip" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 20a1cefe258c..279d0f78ea61 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -10,5 +10,16 @@ resource "google_cloudfunctions_function" "cloud_function" { # Get the source code of the cloud function as a Zip compression trigger_http = true # Name of the function that will be executed when the Google Cloud Function is triggered - entry_point = "init" + + entry_point = "getContentTree" +} + +# Create IAM entry so all users can invoke the function +resource "google_cloudfunctions_function_iam_member" "invoker" { + project = google_cloudfunctions_function.cloud_function.project + region = google_cloudfunctions_function.cloud_function.region + cloud_function = google_cloudfunctions_function.cloud_function.name + + role = "roles/cloudfunctions.invoker" + member = "allUsers" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index f1082c78c570..33d7dc5e0274 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -11,4 +11,5 @@ resource "google_project_iam_member" "terraform_service_account_roles" { role = each.key member = "serviceAccount:${google_service_account.tourofbeam_cf_sa.email}" project = var.project_id -} \ No newline at end of file +} + From 376cab726546d2ccc39c2f8728b9885669dedc3e Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 06:51:47 -0700 Subject: [PATCH 169/269] Update variables.tf --- .../tour-of-beam/terraform-v2/modules/buckets/variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf index 924de05f0e8c..fee8a48c02da 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -14,6 +14,6 @@ variable "project_id" { data "archive_file" "source" { type = "zip" - source_dir = "../../../backend_" - output_path = "/tmp/backend.zip" + source_dir = "../../../backend_v2" + output_path = "/tmp/backend_v2.zip" } \ No newline at end of file From 56eae8417fbc9e215ac2abb2f1d92551090e39af Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 06:52:33 -0700 Subject: [PATCH 170/269] Update main.tf --- .../tour-of-beam/terraform-v2/modules/cloud_functions/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 279d0f78ea61..843f69ab106e 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -11,7 +11,7 @@ resource "google_cloudfunctions_function" "cloud_function" { trigger_http = true # Name of the function that will be executed when the Google Cloud Function is triggered - entry_point = "getContentTree" + entry_point = "sdkList" } # Create IAM entry so all users can invoke the function From c783e6ef959f35d3928cd5ad657e75ce184cc2ae Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 07:42:13 -0700 Subject: [PATCH 171/269] Updates --- learning/tour-of-beam/backend_v2/function.go | 188 ++---------------- .../tour-of-beam/backend_v2/function.go_old | 178 +++++++++++++++++ learning/tour-of-beam/backend_v2/test.txt | 1 + .../modules/cloud_functions/main.tf | 3 +- .../terraform-v2/modules/iam/main.tf | 4 +- 5 files changed, 197 insertions(+), 177 deletions(-) create mode 100644 learning/tour-of-beam/backend_v2/function.go_old create mode 100644 learning/tour-of-beam/backend_v2/test.txt diff --git a/learning/tour-of-beam/backend_v2/function.go b/learning/tour-of-beam/backend_v2/function.go index f771e2eb672f..0c038fb7a2a1 100644 --- a/learning/tour-of-beam/backend_v2/function.go +++ b/learning/tour-of-beam/backend_v2/function.go @@ -1,178 +1,18 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package tob +package api import ( - "context" - "encoding/json" - "fmt" - "log" - "net/http" - "os" - - tob "beam.apache.org/learning/tour-of-beam/backend/internal" - "beam.apache.org/learning/tour-of-beam/backend/internal/service" - "beam.apache.org/learning/tour-of-beam/backend/internal/storage" - "cloud.google.com/go/datastore" - "github.com/GoogleCloudPlatform/functions-framework-go/functions" -) - -const ( - BAD_FORMAT = "BAD_FORMAT" - INTERNAL_ERROR = "INTERNAL_ERROR" - NOT_FOUND = "NOT_FOUND" + "io/ioutil" + "net/http" ) -// Middleware-maker for setting a header -// We also make this less generic: it works with HandlerFunc's -// so that to be convertible to func(w http ResponseWriter, r *http.Request) -// and be accepted by functions.HTTP. -func AddHeader(header, value string) func(http.HandlerFunc) http.HandlerFunc { - return func(next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Add(header, value) - next(w, r) - } - } -} - -// Middleware to check http method. -func EnsureMethod(method string) func(http.HandlerFunc) http.HandlerFunc { - return func(next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if r.Method == method { - next(w, r) - } else { - w.WriteHeader(http.StatusMethodNotAllowed) - } - } - } -} - -// HandleFunc enriched with sdk. -type HandlerFuncWithSdk func(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) - -// middleware to parse sdk query param and pass it as additional handler param. -func ParseSdkParam(next HandlerFuncWithSdk) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - sdkStr := r.URL.Query().Get("sdk") - sdk := tob.ParseSdk(sdkStr) - - if sdk == tob.SDK_UNDEFINED { - log.Printf("Bad sdk: %v", sdkStr) - - message := fmt.Sprintf("Sdk not in: %v", tob.SdksList()) - finalizeErrResponse(w, http.StatusBadRequest, BAD_FORMAT, message) - - return - } - - next(w, r, sdk) - } -} - -// Helper to format http error messages. -func finalizeErrResponse(w http.ResponseWriter, status int, code, message string) { - resp := tob.CodeMessage{Code: code, Message: message} - - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(resp) -} - -var svc service.IContent - -func init() { - // dependencies - // required: - // * TOB_MOCK: respond with static samples - // OR - // * DATASTORE_PROJECT_ID: cloud project id - // optional: - // * DATASTORE_EMULATOR_HOST: emulator host/port (ex. 0.0.0.0:8888) - if os.Getenv("TOB_MOCK") > "" { - svc = &service.Mock{} - } else { - // consumes DATASTORE_* env variables - client, err := datastore.NewClient(context.Background(), "") - if err != nil { - log.Fatalf("new datastore client: %v", err) - } - svc = &service.Svc{Repo: &storage.DatastoreDb{Client: client}} - } - - addHeader := AddHeader("Content-Type", "application/json") - ensureGet := EnsureMethod(http.MethodGet) - - // functions framework - functions.HTTP("sdkList", ensureGet(addHeader(sdkList))) - functions.HTTP("getContentTree", ensureGet(addHeader(ParseSdkParam(getContentTree)))) - functions.HTTP("getUnitContent", ensureGet(addHeader(ParseSdkParam(getUnitContent)))) -} - -// Get list of SDK names -// Used in both representation and accessing content. -func sdkList(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"names": ["Java", "Python", "Go"]}`) -} - -// Get the content tree for a given SDK and user -// Merges info from the default tree and per-user information: -// user code snippets and progress -// Required to be wrapped into ParseSdkParam middleware. -func getContentTree(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { - tree, err := svc.GetContentTree(r.Context(), sdk, nil /*TODO userId*/) - if err != nil { - log.Println("Get content tree error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") - return - } - - err = json.NewEncoder(w).Encode(tree) - if err != nil { - log.Println("Format content tree error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format content tree") - return - } -} - -// Get unit content -// Everything needed to render a learning unit: -// description, hints, code snippets -// Required to be wrapped into ParseSdkParam middleware. -func getUnitContent(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { - unitId := r.URL.Query().Get("unitId") - - unit, err := svc.GetUnitContent(r.Context(), sdk, unitId, nil /*TODO userId*/) - if err != nil { - log.Println("Get unit content error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") - return - } - if unit == nil { - log.Println("Get unit content error:", err) - finalizeErrResponse(w, http.StatusNotFound, NOT_FOUND, "unit not found") - return - } - - err = json.NewEncoder(w).Encode(unit) - if err != nil { - log.Println("Format unit content error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format unit content") - return - } -} +// FileTest func +func FileTest(w http.ResponseWriter, r *http.Request) { + content, err := ioutil.ReadFile("./test.txt") + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + w.Write(content) +} \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/function.go_old b/learning/tour-of-beam/backend_v2/function.go_old new file mode 100644 index 000000000000..5dab8c35f711 --- /dev/null +++ b/learning/tour-of-beam/backend_v2/function.go_old @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tob + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "os" + + tob "beam.apache.org/learning/tour-of-beam/backend/internal" + "beam.apache.org/learning/tour-of-beam/backend/internal/service" + "beam.apache.org/learning/tour-of-beam/backend/internal/storage" + "cloud.google.com/go/datastore" + "github.com/GoogleCloudPlatform/functions-framework-go/functions" +) + +const ( + BAD_FORMAT = "BAD_FORMAT" + INTERNAL_ERROR = "INTERNAL_ERROR" + NOT_FOUND = "NOT_FOUND" +) + +// Middleware-maker for setting a header +// We also make this less generic: it works with HandlerFunc's +// so that to be convertible to func(w http ResponseWriter, r *http.Request) +// and be accepted by functions.HTTP. +func AddHeader(header, value string) func(http.HandlerFunc) http.HandlerFunc { + return func(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Add(header, value) + next(w, r) + } + }g +} + +// Middleware to check http method. +func EnsureMethod(method string) func(http.HandlerFunc) http.HandlerFunc { + return func(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method == method { + next(w, r) + } else { + w.WriteHeader(http.StatusMethodNotAllowed) + } + } + } +} + +// HandleFunc enriched with sdk. +type HandlerFuncWithSdk func(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) + +// middleware to parse sdk query param and pass it as additional handler param. +func ParseSdkParam(next HandlerFuncWithSdk) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + sdkStr := r.URL.Query().Get("sdk") + sdk := tob.ParseSdk(sdkStr) + + if sdk == tob.SDK_UNDEFINED { + log.Printf("Bad sdk: %v", sdkStr) + + message := fmt.Sprintf("Sdk not in: %v", tob.SdksList()) + finalizeErrResponse(w, http.StatusBadRequest, BAD_FORMAT, message) + + return + } + + next(w, r, sdk) + } +} + +// Helper to format http error messages. +func finalizeErrResponse(w http.ResponseWriter, status int, code, message string) { + resp := tob.CodeMessage{Code: code, Message: message} + + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(resp) +} + +var svc service.IContent + +func init() { + // dependencies + // required: + // * TOB_MOCK: respond with static samples + // OR + // * DATASTORE_PROJECT_ID: cloud project id + // optional: + // * DATASTORE_EMULATOR_HOST: emulator host/port (ex. 0.0.0.0:8888) + if os.Getenv("TOB_MOCK") > "" { + svc = &service.Mock{} + } else { + // consumes DATASTORE_* env variables + client, err := datastore.NewClient(context.Background(), "") + if err != nil { + log.Fatalf("new datastore client: %v", err) + } + svc = &service.Svc{Repo: &storage.DatastoreDb{Client: client}} + } + + addHeader := AddHeader("Content-Type", "application/json") + ensureGet := EnsureMethod(http.MethodGet) + + // functions framework + functions.HTTP("sdkList", ensureGet(addHeader(sdkList))) + functions.HTTP("getContentTree", ensureGet(addHeader(ParseSdkParam(getContentTree)))) + functions.HTTP("getUnitContent", ensureGet(addHeader(ParseSdkParam(getUnitContent)))) +} + +// Get list of SDK names +// Used in both representation and accessing content. +func sdkList(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"names": ["Java", "Python", "Go"]}`) +} + +// Get the content tree for a given SDK and user +// Merges info from the default tree and per-user information: +// user code snippets and progress +// Required to be wrapped into ParseSdkParam middleware. +func getContentTree(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { + tree, err := svc.GetContentTree(r.Context(), sdk, nil /*TODO userId*/) + if err != nil { + log.Println("Get content tree error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") + return + } + + err = json.NewEncoder(w).Encode(tree) + if err != nil { + log.Println("Format content tree error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format content tree") + return + } +} + +// Get unit content +// Everything needed to render a learning unit: +// description, hints, code snippets +// Required to be wrapped into ParseSdkParam middleware. +func getUnitContent(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { + unitId := r.URL.Query().Get("unitId") + + unit, err := svc.GetUnitContent(r.Context(), sdk, unitId, nil /*TODO userId*/) + if err != nil { + log.Println("Get unit content error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") + return + } + if unit == nil { + log.Println("Get unit content error:", err) + finalizeErrResponse(w, http.StatusNotFound, NOT_FOUND, "unit not found") + return + } + + err = json.NewEncoder(w).Encode(unit) + if err != nil { + log.Println("Format unit content error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format unit content") + return + } +} diff --git a/learning/tour-of-beam/backend_v2/test.txt b/learning/tour-of-beam/backend_v2/test.txt new file mode 100644 index 000000000000..08c9db954f6e --- /dev/null +++ b/learning/tour-of-beam/backend_v2/test.txt @@ -0,0 +1 @@ +Hello There! \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 843f69ab106e..85c22d680ba0 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -10,8 +10,9 @@ resource "google_cloudfunctions_function" "cloud_function" { # Get the source code of the cloud function as a Zip compression trigger_http = true # Name of the function that will be executed when the Google Cloud Function is triggered + timeout = 500 - entry_point = "sdkList" + entry_point = "FileTest" } # Create IAM entry so all users can invoke the function diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index 33d7dc5e0274..3eefd5ff3e76 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -5,11 +5,11 @@ resource "google_service_account" "tourofbeam_cf_sa" { resource "google_project_iam_member" "terraform_service_account_roles" { for_each = toset([ - "roles/resourcemanager.projectIamAdmin", "roles/cloudfunctions.developer", "roles/storage.objectViewer", + "roles/resourcemanager.projectIamAdmin", "roles/cloudfunctions.admin", "roles/storage.objectViewer", "roles/storage.objectCreator", "roles/iam.serviceAccountAdmin", ]) role = each.key member = "serviceAccount:${google_service_account.tourofbeam_cf_sa.email}" project = var.project_id } - +g From d68f873d6cea4276b114f8c646397b9f2feb6994 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 08:24:46 -0700 Subject: [PATCH 172/269] Updates --- learning/tour-of-beam/backend_v2/function.go | 188 ++++++++++++++++-- .../tour-of-beam/backend_v2/function.go_old | 178 ----------------- .../modules/cloud_functions/main.tf | 4 +- .../terraform-v2/modules/iam/main.tf | 2 +- 4 files changed, 176 insertions(+), 196 deletions(-) delete mode 100644 learning/tour-of-beam/backend_v2/function.go_old diff --git a/learning/tour-of-beam/backend_v2/function.go b/learning/tour-of-beam/backend_v2/function.go index 0c038fb7a2a1..f771e2eb672f 100644 --- a/learning/tour-of-beam/backend_v2/function.go +++ b/learning/tour-of-beam/backend_v2/function.go @@ -1,18 +1,178 @@ -package api +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tob import ( - "io/ioutil" - "net/http" + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "os" + + tob "beam.apache.org/learning/tour-of-beam/backend/internal" + "beam.apache.org/learning/tour-of-beam/backend/internal/service" + "beam.apache.org/learning/tour-of-beam/backend/internal/storage" + "cloud.google.com/go/datastore" + "github.com/GoogleCloudPlatform/functions-framework-go/functions" +) + +const ( + BAD_FORMAT = "BAD_FORMAT" + INTERNAL_ERROR = "INTERNAL_ERROR" + NOT_FOUND = "NOT_FOUND" ) -// FileTest func -func FileTest(w http.ResponseWriter, r *http.Request) { - content, err := ioutil.ReadFile("./test.txt") - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) - return - } - - w.Write(content) -} \ No newline at end of file +// Middleware-maker for setting a header +// We also make this less generic: it works with HandlerFunc's +// so that to be convertible to func(w http ResponseWriter, r *http.Request) +// and be accepted by functions.HTTP. +func AddHeader(header, value string) func(http.HandlerFunc) http.HandlerFunc { + return func(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Add(header, value) + next(w, r) + } + } +} + +// Middleware to check http method. +func EnsureMethod(method string) func(http.HandlerFunc) http.HandlerFunc { + return func(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method == method { + next(w, r) + } else { + w.WriteHeader(http.StatusMethodNotAllowed) + } + } + } +} + +// HandleFunc enriched with sdk. +type HandlerFuncWithSdk func(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) + +// middleware to parse sdk query param and pass it as additional handler param. +func ParseSdkParam(next HandlerFuncWithSdk) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + sdkStr := r.URL.Query().Get("sdk") + sdk := tob.ParseSdk(sdkStr) + + if sdk == tob.SDK_UNDEFINED { + log.Printf("Bad sdk: %v", sdkStr) + + message := fmt.Sprintf("Sdk not in: %v", tob.SdksList()) + finalizeErrResponse(w, http.StatusBadRequest, BAD_FORMAT, message) + + return + } + + next(w, r, sdk) + } +} + +// Helper to format http error messages. +func finalizeErrResponse(w http.ResponseWriter, status int, code, message string) { + resp := tob.CodeMessage{Code: code, Message: message} + + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(resp) +} + +var svc service.IContent + +func init() { + // dependencies + // required: + // * TOB_MOCK: respond with static samples + // OR + // * DATASTORE_PROJECT_ID: cloud project id + // optional: + // * DATASTORE_EMULATOR_HOST: emulator host/port (ex. 0.0.0.0:8888) + if os.Getenv("TOB_MOCK") > "" { + svc = &service.Mock{} + } else { + // consumes DATASTORE_* env variables + client, err := datastore.NewClient(context.Background(), "") + if err != nil { + log.Fatalf("new datastore client: %v", err) + } + svc = &service.Svc{Repo: &storage.DatastoreDb{Client: client}} + } + + addHeader := AddHeader("Content-Type", "application/json") + ensureGet := EnsureMethod(http.MethodGet) + + // functions framework + functions.HTTP("sdkList", ensureGet(addHeader(sdkList))) + functions.HTTP("getContentTree", ensureGet(addHeader(ParseSdkParam(getContentTree)))) + functions.HTTP("getUnitContent", ensureGet(addHeader(ParseSdkParam(getUnitContent)))) +} + +// Get list of SDK names +// Used in both representation and accessing content. +func sdkList(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"names": ["Java", "Python", "Go"]}`) +} + +// Get the content tree for a given SDK and user +// Merges info from the default tree and per-user information: +// user code snippets and progress +// Required to be wrapped into ParseSdkParam middleware. +func getContentTree(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { + tree, err := svc.GetContentTree(r.Context(), sdk, nil /*TODO userId*/) + if err != nil { + log.Println("Get content tree error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") + return + } + + err = json.NewEncoder(w).Encode(tree) + if err != nil { + log.Println("Format content tree error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format content tree") + return + } +} + +// Get unit content +// Everything needed to render a learning unit: +// description, hints, code snippets +// Required to be wrapped into ParseSdkParam middleware. +func getUnitContent(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { + unitId := r.URL.Query().Get("unitId") + + unit, err := svc.GetUnitContent(r.Context(), sdk, unitId, nil /*TODO userId*/) + if err != nil { + log.Println("Get unit content error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") + return + } + if unit == nil { + log.Println("Get unit content error:", err) + finalizeErrResponse(w, http.StatusNotFound, NOT_FOUND, "unit not found") + return + } + + err = json.NewEncoder(w).Encode(unit) + if err != nil { + log.Println("Format unit content error:", err) + finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format unit content") + return + } +} diff --git a/learning/tour-of-beam/backend_v2/function.go_old b/learning/tour-of-beam/backend_v2/function.go_old deleted file mode 100644 index 5dab8c35f711..000000000000 --- a/learning/tour-of-beam/backend_v2/function.go_old +++ /dev/null @@ -1,178 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package tob - -import ( - "context" - "encoding/json" - "fmt" - "log" - "net/http" - "os" - - tob "beam.apache.org/learning/tour-of-beam/backend/internal" - "beam.apache.org/learning/tour-of-beam/backend/internal/service" - "beam.apache.org/learning/tour-of-beam/backend/internal/storage" - "cloud.google.com/go/datastore" - "github.com/GoogleCloudPlatform/functions-framework-go/functions" -) - -const ( - BAD_FORMAT = "BAD_FORMAT" - INTERNAL_ERROR = "INTERNAL_ERROR" - NOT_FOUND = "NOT_FOUND" -) - -// Middleware-maker for setting a header -// We also make this less generic: it works with HandlerFunc's -// so that to be convertible to func(w http ResponseWriter, r *http.Request) -// and be accepted by functions.HTTP. -func AddHeader(header, value string) func(http.HandlerFunc) http.HandlerFunc { - return func(next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Add(header, value) - next(w, r) - } - }g -} - -// Middleware to check http method. -func EnsureMethod(method string) func(http.HandlerFunc) http.HandlerFunc { - return func(next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if r.Method == method { - next(w, r) - } else { - w.WriteHeader(http.StatusMethodNotAllowed) - } - } - } -} - -// HandleFunc enriched with sdk. -type HandlerFuncWithSdk func(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) - -// middleware to parse sdk query param and pass it as additional handler param. -func ParseSdkParam(next HandlerFuncWithSdk) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - sdkStr := r.URL.Query().Get("sdk") - sdk := tob.ParseSdk(sdkStr) - - if sdk == tob.SDK_UNDEFINED { - log.Printf("Bad sdk: %v", sdkStr) - - message := fmt.Sprintf("Sdk not in: %v", tob.SdksList()) - finalizeErrResponse(w, http.StatusBadRequest, BAD_FORMAT, message) - - return - } - - next(w, r, sdk) - } -} - -// Helper to format http error messages. -func finalizeErrResponse(w http.ResponseWriter, status int, code, message string) { - resp := tob.CodeMessage{Code: code, Message: message} - - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(resp) -} - -var svc service.IContent - -func init() { - // dependencies - // required: - // * TOB_MOCK: respond with static samples - // OR - // * DATASTORE_PROJECT_ID: cloud project id - // optional: - // * DATASTORE_EMULATOR_HOST: emulator host/port (ex. 0.0.0.0:8888) - if os.Getenv("TOB_MOCK") > "" { - svc = &service.Mock{} - } else { - // consumes DATASTORE_* env variables - client, err := datastore.NewClient(context.Background(), "") - if err != nil { - log.Fatalf("new datastore client: %v", err) - } - svc = &service.Svc{Repo: &storage.DatastoreDb{Client: client}} - } - - addHeader := AddHeader("Content-Type", "application/json") - ensureGet := EnsureMethod(http.MethodGet) - - // functions framework - functions.HTTP("sdkList", ensureGet(addHeader(sdkList))) - functions.HTTP("getContentTree", ensureGet(addHeader(ParseSdkParam(getContentTree)))) - functions.HTTP("getUnitContent", ensureGet(addHeader(ParseSdkParam(getUnitContent)))) -} - -// Get list of SDK names -// Used in both representation and accessing content. -func sdkList(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"names": ["Java", "Python", "Go"]}`) -} - -// Get the content tree for a given SDK and user -// Merges info from the default tree and per-user information: -// user code snippets and progress -// Required to be wrapped into ParseSdkParam middleware. -func getContentTree(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { - tree, err := svc.GetContentTree(r.Context(), sdk, nil /*TODO userId*/) - if err != nil { - log.Println("Get content tree error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") - return - } - - err = json.NewEncoder(w).Encode(tree) - if err != nil { - log.Println("Format content tree error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format content tree") - return - } -} - -// Get unit content -// Everything needed to render a learning unit: -// description, hints, code snippets -// Required to be wrapped into ParseSdkParam middleware. -func getUnitContent(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { - unitId := r.URL.Query().Get("unitId") - - unit, err := svc.GetUnitContent(r.Context(), sdk, unitId, nil /*TODO userId*/) - if err != nil { - log.Println("Get unit content error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") - return - } - if unit == nil { - log.Println("Get unit content error:", err) - finalizeErrResponse(w, http.StatusNotFound, NOT_FOUND, "unit not found") - return - } - - err = json.NewEncoder(w).Encode(unit) - if err != nil { - log.Println("Format unit content error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format unit content") - return - } -} diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 85c22d680ba0..530a6a8d6ee9 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -10,9 +10,7 @@ resource "google_cloudfunctions_function" "cloud_function" { # Get the source code of the cloud function as a Zip compression trigger_http = true # Name of the function that will be executed when the Google Cloud Function is triggered - timeout = 500 - - entry_point = "FileTest" + entry_point = "sdkList" } # Create IAM entry so all users can invoke the function diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index 3eefd5ff3e76..d91e46e16b0d 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -12,4 +12,4 @@ resource "google_project_iam_member" "terraform_service_account_roles" { member = "serviceAccount:${google_service_account.tourofbeam_cf_sa.email}" project = var.project_id } -g + From 45a7d801cdd7d505b0209f24d6992df40708458d Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 14 Oct 2022 08:43:28 -0700 Subject: [PATCH 173/269] Updates --- .../terraform-v2/modules/buckets/variables.tf | 4 ++-- .../terraform-v2/modules/cloud_functions/main.tf | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf index fee8a48c02da..0416cc20cbfd 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -14,6 +14,6 @@ variable "project_id" { data "archive_file" "source" { type = "zip" - source_dir = "../../../backend_v2" - output_path = "/tmp/backend_v2.zip" + source_dir = "../../../backend" + output_path = "/tmp/backend.zip" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 530a6a8d6ee9..1b765ee7bac0 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -10,7 +10,13 @@ resource "google_cloudfunctions_function" "cloud_function" { # Get the source code of the cloud function as a Zip compression trigger_http = true # Name of the function that will be executed when the Google Cloud Function is triggered - entry_point = "sdkList" + entry_point = "getSdkList" + + environment_variables = { + DATASTORE_PROJECT_ID="test-cloud-func-deploy" + TOB_MOCK=1 + } + } # Create IAM entry so all users can invoke the function From 0718b1fa6ddf9483b3b07022bcdbe025dcc66e17 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Sun, 16 Oct 2022 13:05:44 -0700 Subject: [PATCH 174/269] Updates --- learning/tour-of-beam/backend_v2/README.md | 56 -- .../backend_v2/cmd/ci_cd/ci_cd.go | 54 -- learning/tour-of-beam/backend_v2/cmd/main.go | 36 - .../WEB-INF/appengine-generated/local_db.bin | Bin 6927 -> 0 bytes .../backend_v2/datadir/WEB-INF/index.yaml | 20 - .../tour-of-beam/backend_v2/datadir/env.yaml | 6 - .../backend_v2/docker-compose.yml | 28 - learning/tour-of-beam/backend_v2/function.go | 178 ----- learning/tour-of-beam/backend_v2/go.mod | 28 - learning/tour-of-beam/backend_v2/go.sum | 688 ------------------ .../backend_v2/internal/entity.go | 69 -- .../internal/fs_content/builders.go | 45 -- .../backend_v2/internal/fs_content/load.go | 210 ------ .../internal/fs_content/load_test.go | 79 -- .../backend_v2/internal/fs_content/yaml.go | 81 --- .../tour-of-beam/backend_v2/internal/json.go | 29 - .../tour-of-beam/backend_v2/internal/sdk.go | 51 -- .../backend_v2/internal/sdk_test.go | 59 -- .../backend_v2/internal/service/content.go | 42 -- .../backend_v2/internal/service/mock.go | 48 -- .../backend_v2/internal/storage/adapter.go | 146 ---- .../backend_v2/internal/storage/datastore.go | 260 ------- .../backend_v2/internal/storage/iface.go | 29 - .../internal/storage/image/Dockerfile | 32 - .../internal/storage/image/start-datastore.sh | 61 -- .../backend_v2/internal/storage/index.yaml | 20 - .../backend_v2/internal/storage/schema.go | 99 --- .../samples/api/get_content_tree.json | 41 -- .../samples/api/get_unit_content.json | 10 - .../learning-content/java/content-info.yaml | 4 - .../java/module 1/module-info.yaml | 7 - .../module 1/unit-challenge/description.md | 3 - .../java/module 1/unit-challenge/hint1.md | 3 - .../java/module 1/unit-challenge/hint2.md | 3 - .../module 1/unit-challenge/unit-info.yaml | 5 - .../java/module 1/unit-example/unit-info.yaml | 3 - .../java/module 2/module-info.yaml | 7 - .../module 2/unit-challenge/description.md | 3 - .../java/module 2/unit-challenge/hint1.md | 3 - .../java/module 2/unit-challenge/hint2.md | 3 - .../module 2/unit-challenge/unit-info.yaml | 5 - .../java/module 2/unit-example/unit-info.yaml | 3 - .../learning-content/python/content-info.yaml | 3 - .../python/module 1/group/group-info.yaml | 7 - .../group/unit-challenge/description.md | 3 - .../module 1/group/unit-challenge/hint1.md | 3 - .../module 1/group/unit-challenge/hint2.md | 3 - .../group/unit-challenge/unit-info.yaml | 5 - .../group/unit-example/unit-info.yaml | 3 - .../python/module 1/intro-unit/unit-info.yaml | 3 - .../python/module 1/module-info.yaml | 7 - .../java/content-info.yaml | 2 - .../python/content-info.yaml | 2 - learning/tour-of-beam/backend_v2/test.txt | 1 - learning/tour-of-beam/terraform-v2/README.md | 56 ++ .../terraform-v2/environments/uat/outputs.tf | 12 +- .../modules/cloud_functions/main.tf | 80 +- .../modules/cloud_functions/outputs.tf | 12 +- .../terraform-v2/modules/iam/main.tf | 2 +- 59 files changed, 145 insertions(+), 2616 deletions(-) delete mode 100644 learning/tour-of-beam/backend_v2/README.md delete mode 100644 learning/tour-of-beam/backend_v2/cmd/ci_cd/ci_cd.go delete mode 100644 learning/tour-of-beam/backend_v2/cmd/main.go delete mode 100644 learning/tour-of-beam/backend_v2/datadir/WEB-INF/appengine-generated/local_db.bin delete mode 100644 learning/tour-of-beam/backend_v2/datadir/WEB-INF/index.yaml delete mode 100644 learning/tour-of-beam/backend_v2/datadir/env.yaml delete mode 100644 learning/tour-of-beam/backend_v2/docker-compose.yml delete mode 100644 learning/tour-of-beam/backend_v2/function.go delete mode 100644 learning/tour-of-beam/backend_v2/go.mod delete mode 100644 learning/tour-of-beam/backend_v2/go.sum delete mode 100644 learning/tour-of-beam/backend_v2/internal/entity.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/fs_content/builders.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/fs_content/load.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/fs_content/load_test.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/fs_content/yaml.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/json.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/sdk.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/sdk_test.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/service/content.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/service/mock.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/storage/adapter.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/storage/datastore.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/storage/iface.go delete mode 100644 learning/tour-of-beam/backend_v2/internal/storage/image/Dockerfile delete mode 100644 learning/tour-of-beam/backend_v2/internal/storage/image/start-datastore.sh delete mode 100644 learning/tour-of-beam/backend_v2/internal/storage/index.yaml delete mode 100644 learning/tour-of-beam/backend_v2/internal/storage/schema.go delete mode 100644 learning/tour-of-beam/backend_v2/samples/api/get_content_tree.json delete mode 100644 learning/tour-of-beam/backend_v2/samples/api/get_unit_content.json delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/content-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/module-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/description.md delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint1.md delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint2.md delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-example/unit-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/module-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/description.md delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint1.md delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint2.md delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-example/unit-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/content-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/group-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/description.md delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint1.md delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint2.md delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/intro-unit/unit-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/module-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content_empty/java/content-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/samples/learning-content_empty/python/content-info.yaml delete mode 100644 learning/tour-of-beam/backend_v2/test.txt create mode 100644 learning/tour-of-beam/terraform-v2/README.md diff --git a/learning/tour-of-beam/backend_v2/README.md b/learning/tour-of-beam/backend_v2/README.md deleted file mode 100644 index 2be311203b2c..000000000000 --- a/learning/tour-of-beam/backend_v2/README.md +++ /dev/null @@ -1,56 +0,0 @@ - - -## Tour Of Beam Backend - -Backend provides the learning content tree for a given SDK, -and currently logged-in user's snippets and progress. -Currently it supports Java, Python, and Go Beam SDK. - -It is comprised of several Cloud Functions, with Firerstore in Datastore mode as a storage. -* list-sdks -* get-content-tree?sdk=(Java|Go|Python) -* get-unit-content?unitId= -TODO: add response schemas -TODO: add save functions info -TODO: add user token info - -### Datastore schema - -The storage is shared with Beam Playground backend, so that Tour Of Beam could access its entities in -pg_examples and pg_snippets. - -Conceptually, the learning tree is a tree with a root node in tb_learning_path, -having several children of tb_learning_module, and each module has its descendant nodes. -Node is either a group or a unit. - -Every module or unit has SDK-wide unique ID, which is provided by a content maintainer. -User's progress on a unit is tied to its ID, and if ID changes, the progress is lost. - -__Kinds__ -- tb_learning_path - - key: `(SDK_JAVA|SDK_PYTHON|SDK_GO)` - -- tb_learning_module - - key: `_` - - parentKey: Learning Path key SDK - -- tb_learning_node - - key: `_` - - parentKey: parent module/group key diff --git a/learning/tour-of-beam/backend_v2/cmd/ci_cd/ci_cd.go b/learning/tour-of-beam/backend_v2/cmd/ci_cd/ci_cd.go deleted file mode 100644 index 9e7e109fecf0..000000000000 --- a/learning/tour-of-beam/backend_v2/cmd/ci_cd/ci_cd.go +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "log" - "os" - - "beam.apache.org/learning/tour-of-beam/backend/internal/fs_content" - "beam.apache.org/learning/tour-of-beam/backend/internal/storage" - "cloud.google.com/go/datastore" -) - -var ( - repo storage.Iface - ctx context.Context -) - -func init() { - ctx = context.Background() - client, err := datastore.NewClient(ctx, "") - if err != nil { - log.Fatalf("new datastore client: %v", err) - } - repo = &storage.DatastoreDb{Client: client} -} - -func main() { - learningRoot := os.Getenv("TOB_LEARNING_ROOT") - log.Printf("Parsing learning-content at %q\n", learningRoot) - trees, err := fs_content.CollectLearningTree(learningRoot) - if err != nil { - log.Fatal(err) - } - - log.Printf("collected %v sdks\n", len(trees)) - if err = repo.SaveContentTrees(ctx, trees); err != nil { - log.Fatal(err) - } -} diff --git a/learning/tour-of-beam/backend_v2/cmd/main.go b/learning/tour-of-beam/backend_v2/cmd/main.go deleted file mode 100644 index b2b742534c68..000000000000 --- a/learning/tour-of-beam/backend_v2/cmd/main.go +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "log" - "os" - - // Blank-import the function package so the init() runs. - _ "beam.apache.org/learning/tour-of-beam/backend" - "github.com/GoogleCloudPlatform/functions-framework-go/funcframework" -) - -func main() { - // Use PORT environment variable, or default to 8080. - port := "8080" - if envPort := os.Getenv("PORT"); envPort != "" { - port = envPort - } - if err := funcframework.Start(port); err != nil { - log.Fatalf("funcframework.Start: %v\n", err) - } -} diff --git a/learning/tour-of-beam/backend_v2/datadir/WEB-INF/appengine-generated/local_db.bin b/learning/tour-of-beam/backend_v2/datadir/WEB-INF/appengine-generated/local_db.bin deleted file mode 100644 index ec17372c131cdf13aae1b935fa7b5b1f1bc60c00..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6927 zcmds*-EZ4e6u__DWKGj_OVY1#2b0xpf>m0o_JwwPV4r1O)2&jwNx*~WCU=RANgDdGiAafK z-{b3h&iVPAd(N$2L3DNZKh@9Qp!(swqd~v{MY6aR5n4nI-*KiVdC71LMJ>@ z*6D*%^#`4nj3T`$+WXU;Z$~bEPbZxPaTQDXSt(~?G6(w1B0)wGtNF72%u#o>iPacA zCh3Qe z&dMrUGOd;U&YwksFVVx-)1zPh^26Y59&lOEOVvS|X=83n(6ejnsChw08fuGiva$GvUwFa)VRq zwkM_jG^4aGnZ>c#(#d)8%&`l{68-6P&?F_Cw^THi^v1>Ng-I&M%;K1CEagj~vP6A{ zCKDfV@kLdt6bQDoJQ;vdO!9~bV4RNu=Tt>kkXk!Uz=&DVk$c9GH00$x{U3)hZ#MD- zjA&BHkr;%jQ`Kh(=QY_B&eF?dG?s+@qG;qK(?moRsayo|t3MM6)1v4ax+2!h=#KI+ zo*aN9{#eyXqDj5vj`zjx-u>kEclXy=v8s#0#O!yu*dUBp&XU0Rbq>DDDG>z_V27w< zw^zp=g#DDq9FcYB^``Uj5ZJ?TfW9X$E@`r^BXiENkaPJlm@=?R-)yWk3|-^EHEwT% zyBeHsIZW-Ou)MFl!N7vs-?tH127Op^MTy2Rth=ks6pyc{l2H5F2Vt}* z(}*c><9p%b^q$l7Oqh(vC0$o5LIqpIKrhPlDArP86NpUU>yTLoYZmQ?eg1TvQtS4S_+BuB-r(SC9*JSgegsqK zZRHCFix0UlF5-?y7oS@`eeP_FEtl)_szm2aS53~@X6fO3v*G#!ML$$EhBIbE#u0Wy z7Mhc~33aU*>glix`)2La_3XktF}r16{eNaqJJdC9FcUh#=WqR%l+PG$KX^2_b+?H% zyuCW0i>jg#;*f24r_dL^Ppn*RvrpV?o?BKGT~`B}Rflv37x$I&76aGiPFRLl78{6( zY#|zqY&;2MmgOQ6o^_%q8TQKznkVa*nMB)EVT7|Xb8j12nV=;_)J+7mG#rCMt^N#G zr#m7z_w9g@Gqj`G zqVXd>)y8{lN2j^*^U9rdHGVv?b+=f+1Lbf~^N!|abx-@c$KZ~|idn>33+)TPx6yu+ z6Vx@XqZ&?=xw)~}f|DMP6YhECV$fQ%VF9Px=H*mF1WDEEp+4AK?QTwy5#P+T_q5>l zK!xR1Om@>w6m~kBW!bFI_)49yaO&jT`31JM$+hswkMFecNhcz^Y(=h{Nbcws*toH> zXtMESsEyBQw2Rwd<6am&er##(1Z(1Uk@=xna#qO)X+0B0>wl~K*eI}G6Ln$rZ7a^! Q6D!J2Kf15%9(v0E0^^%w!T "" { - svc = &service.Mock{} - } else { - // consumes DATASTORE_* env variables - client, err := datastore.NewClient(context.Background(), "") - if err != nil { - log.Fatalf("new datastore client: %v", err) - } - svc = &service.Svc{Repo: &storage.DatastoreDb{Client: client}} - } - - addHeader := AddHeader("Content-Type", "application/json") - ensureGet := EnsureMethod(http.MethodGet) - - // functions framework - functions.HTTP("sdkList", ensureGet(addHeader(sdkList))) - functions.HTTP("getContentTree", ensureGet(addHeader(ParseSdkParam(getContentTree)))) - functions.HTTP("getUnitContent", ensureGet(addHeader(ParseSdkParam(getUnitContent)))) -} - -// Get list of SDK names -// Used in both representation and accessing content. -func sdkList(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"names": ["Java", "Python", "Go"]}`) -} - -// Get the content tree for a given SDK and user -// Merges info from the default tree and per-user information: -// user code snippets and progress -// Required to be wrapped into ParseSdkParam middleware. -func getContentTree(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { - tree, err := svc.GetContentTree(r.Context(), sdk, nil /*TODO userId*/) - if err != nil { - log.Println("Get content tree error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") - return - } - - err = json.NewEncoder(w).Encode(tree) - if err != nil { - log.Println("Format content tree error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format content tree") - return - } -} - -// Get unit content -// Everything needed to render a learning unit: -// description, hints, code snippets -// Required to be wrapped into ParseSdkParam middleware. -func getUnitContent(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) { - unitId := r.URL.Query().Get("unitId") - - unit, err := svc.GetUnitContent(r.Context(), sdk, unitId, nil /*TODO userId*/) - if err != nil { - log.Println("Get unit content error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "storage error") - return - } - if unit == nil { - log.Println("Get unit content error:", err) - finalizeErrResponse(w, http.StatusNotFound, NOT_FOUND, "unit not found") - return - } - - err = json.NewEncoder(w).Encode(unit) - if err != nil { - log.Println("Format unit content error:", err) - finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format unit content") - return - } -} diff --git a/learning/tour-of-beam/backend_v2/go.mod b/learning/tour-of-beam/backend_v2/go.mod deleted file mode 100644 index 6601abeee276..000000000000 --- a/learning/tour-of-beam/backend_v2/go.mod +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module beam.apache.org/learning/tour-of-beam/backend - -go 1.16 - -require ( - github.com/GoogleCloudPlatform/functions-framework-go v1.5.3 - gopkg.in/yaml.v3 v3.0.1 -) - -require ( - cloud.google.com/go/datastore v1.8.0 - github.com/stretchr/testify v1.8.0 -) diff --git a/learning/tour-of-beam/backend_v2/go.sum b/learning/tour-of-beam/backend_v2/go.sum deleted file mode 100644 index 45dd6a5e1909..000000000000 --- a/learning/tour-of-beam/backend_v2/go.sum +++ /dev/null @@ -1,688 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.1 h1:vpK6iQWv/2uUeFJth4/cBHsQAGjn1iIE6AAlxipRaA0= -cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.8.0 h1:2qo2G7hABSeqswa+5Ga3+QB8/ZwKOJmDsCISM9scmsU= -cloud.google.com/go/datastore v1.8.0/go.mod h1:q1CpHVByTlXppdqTcu4LIhCsTn3fhtZ5R7+TajciO+M= -cloud.google.com/go/functions v1.0.0 h1:cOFEDJ3sgAFRjRULSUJ0Q8cw9qFa5JdpXIBWoNX5uDw= -cloud.google.com/go/functions v1.0.0/go.mod h1:O9KS8UweFVo6GbbbCBKh5yEzbW08PVkg2spe3RfPMd4= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/functions-framework-go v1.5.3 h1:Xx8uWT4hjgbjuXexbpU6V0yawWOdrbcAzZVyMYJvX8Q= -github.com/GoogleCloudPlatform/functions-framework-go v1.5.3/go.mod h1:pq+lZy4vONJ5fjd3q/B6QzWhfHPAbuVweLpxZzMOb9Y= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudevents/sdk-go/v2 v2.6.1 h1:yHtzgmeBvc0TZx1nrnvYXov1CSvkQyvhEhNMs8Z5Mmk= -github.com/cloudevents/sdk-go/v2 v2.6.1/go.mod h1:nlXhgFkf0uTopxmRXalyMwS2LG70cRGPrxzmjJgSG0U= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa h1:7MYGT2XEMam7Mtzv1yDUYXANedWvwk3HKkR3MyGowy8= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d h1:Zu/JngovGLVi6t2J3nmAf3AoTDwuzw85YZ3b9o4yU7s= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.84.0 h1:NMB9J4cCxs9xEm+1Z9QiO3eFvn7EnQj3Eo3hN6ugVlg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad h1:kqrS+lhvaMHCxul6sKQvKJ8nAAhlVItmZV822hYFH/U= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/learning/tour-of-beam/backend_v2/internal/entity.go b/learning/tour-of-beam/backend_v2/internal/entity.go deleted file mode 100644 index 538715303452..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/entity.go +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -type Unit struct { - Id string `json:"unitId"` - Name string `json:"name"` - - // optional - Description string `json:"description,omitempty"` - Hints []string `json:"hints,omitempty"` - TaskSnippetId string `json:"taskSnippetId,omitempty"` - SolutionSnippetId string `json:"solutionSnippetId,omitempty"` - TaskName string `json:"-"` - SolutionName string `json:"-"` - - // optional, user-specific - UserSnippetId string `json:"userSnippetId,omitempty"` - IsCompleted string `json:"isCompleted,omitempty"` -} - -type NodeType int - -const ( - NODE_UNDEFINED NodeType = iota - NODE_UNIT - NODE_GROUP -) - -type Group struct { - Name string `json:"name"` - Nodes []Node `json:"nodes"` -} - -type Node struct { - Type NodeType `json:"type"` - Group *Group `json:"group,omitempty"` - Unit *Unit `json:"unit,omitempty"` -} - -type Module struct { - Id string `json:"moduleId"` - Name string `json:"name"` - Complexity string `json:"complexity"` - Nodes []Node `json:"nodes"` -} - -type ContentTree struct { - Sdk Sdk `json:"sdk"` - Modules []Module `json:"modules"` -} - -type CodeMessage struct { - Code string `json:"code"` - Message string `json:"message,omitempty"` -} diff --git a/learning/tour-of-beam/backend_v2/internal/fs_content/builders.go b/learning/tour-of-beam/backend_v2/internal/fs_content/builders.go deleted file mode 100644 index 84895431cb08..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/fs_content/builders.go +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fs_content - -import ( - tob "beam.apache.org/learning/tour-of-beam/backend/internal" -) - -type UnitBuilder struct { - tob.Unit -} - -func NewUnitBuilder(info learningUnitInfo) UnitBuilder { - return UnitBuilder{tob.Unit{ - Id: info.Id, - Name: info.Name, - TaskName: info.TaskName, - SolutionName: info.SolutionName, - }} -} - -func (b *UnitBuilder) AddDescription(d string) { - b.Description = d -} - -func (b *UnitBuilder) AddHint(h string) { - b.Hints = append(b.Hints, h) -} - -func (b *UnitBuilder) Build() *tob.Unit { - return &b.Unit -} diff --git a/learning/tour-of-beam/backend_v2/internal/fs_content/load.go b/learning/tour-of-beam/backend_v2/internal/fs_content/load.go deleted file mode 100644 index 8c9ea4271045..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/fs_content/load.go +++ /dev/null @@ -1,210 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fs_content - -import ( - "fmt" - "io/fs" - "io/ioutil" - "log" - "os" - "path/filepath" - "regexp" - - tob "beam.apache.org/learning/tour-of-beam/backend/internal" -) - -const ( - contentInfoYaml = "content-info.yaml" - moduleInfoYaml = "module-info.yaml" - groupInfoYaml = "group-info.yaml" - unitInfoYaml = "unit-info.yaml" - - descriptionMd = "description.md" - hintMdRegexp = "hint[0-9]*.md" -) - -type learningPathInfo struct { - Sdk string - Content []string `yaml:"content"` -} - -type learningModuleInfo struct { - Id string - Name string - Complexity string - Content []string `yaml:"content"` -} - -type learningGroupInfo struct { - Name string - Content []string `yaml:"content"` -} - -type learningUnitInfo struct { - Id string - Name string - TaskName string - SolutionName string -} - -// Watch for duplicate ids. Not thread-safe! -type idsWatcher struct { - ids map[string]struct{} -} - -func (w *idsWatcher) CheckId(id string) { - if _, exists := w.ids[id]; exists { - log.Fatalf("Duplicate id: %v", id) - } - w.ids[id] = struct{}{} -} - -func NewIdsWatcher() idsWatcher { - return idsWatcher{make(map[string]struct{})} -} - -func collectUnit(infopath string, ids_watcher *idsWatcher) (unit *tob.Unit, err error) { - info := loadLearningUnitInfo(infopath) - log.Printf("Found Unit %v metadata at %v\n", info.Id, infopath) - ids_watcher.CheckId(info.Id) - builder := NewUnitBuilder(info) - - rootpath := filepath.Join(infopath, "..") - err = filepath.WalkDir(rootpath, - func(path string, d fs.DirEntry, err error) error { - switch { - // skip nested dirs - case path > rootpath && d.IsDir(): - return filepath.SkipDir - - case d.Name() == descriptionMd: - content, err := ioutil.ReadFile(path) - if err != nil { - return err - } - builder.AddDescription(string(content)) - - // Here we rely on that WalkDir entries are lexically sorted - case regexp.MustCompile(hintMdRegexp).MatchString(d.Name()): - content, err := ioutil.ReadFile(path) - if err != nil { - return err - } - builder.AddHint(string(content)) - } - return nil - }) - - return builder.Build(), err -} - -func collectGroup(infopath string, ids_watcher *idsWatcher) (*tob.Group, error) { - info := loadLearningGroupInfo(infopath) - log.Printf("Found Group %v metadata at %v\n", info.Name, infopath) - group := tob.Group{Name: info.Name} - for _, item := range info.Content { - node, err := collectNode(filepath.Join(infopath, "..", item), ids_watcher) - if err != nil { - return &group, err - } - group.Nodes = append(group.Nodes, node) - } - - return &group, nil -} - -// Collect node which is either a unit or a group. -func collectNode(rootpath string, ids_watcher *idsWatcher) (node tob.Node, err error) { - files, err := os.ReadDir(rootpath) - if err != nil { - return node, err - } - for _, f := range files { - switch f.Name() { - case unitInfoYaml: - node.Type = tob.NODE_UNIT - node.Unit, err = collectUnit(filepath.Join(rootpath, unitInfoYaml), ids_watcher) - case groupInfoYaml: - node.Type = tob.NODE_GROUP - node.Group, err = collectGroup(filepath.Join(rootpath, groupInfoYaml), ids_watcher) - } - } - if node.Type == tob.NODE_UNDEFINED { - return node, fmt.Errorf("node undefined at %v", rootpath) - } - return node, err -} - -func collectModule(infopath string, ids_watcher *idsWatcher) (tob.Module, error) { - info := loadLearningModuleInfo(infopath) - log.Printf("Found Module %v metadata at %v\n", info.Id, infopath) - ids_watcher.CheckId(info.Id) - module := tob.Module{Id: info.Id, Name: info.Name, Complexity: info.Complexity} - for _, item := range info.Content { - node, err := collectNode(filepath.Join(infopath, "..", item), ids_watcher) - if err != nil { - return tob.Module{}, err - } - module.Nodes = append(module.Nodes, node) - } - - return module, nil -} - -func collectSdk(infopath string) (tree tob.ContentTree, err error) { - ids_watcher := NewIdsWatcher() - - info := loadLearningPathInfo(infopath) - tree.Sdk = tob.ParseSdk(info.Sdk) - if tree.Sdk == tob.SDK_UNDEFINED { - return tree, fmt.Errorf("unknown SDK at %v", infopath) - } - log.Printf("Found Sdk %v metadata at %v\n", info.Sdk, infopath) - for _, item := range info.Content { - mod, err := collectModule(filepath.Join(infopath, "..", item, moduleInfoYaml), &ids_watcher) - if err != nil { - return tree, err - } - tree.Modules = append(tree.Modules, mod) - } - - return tree, nil -} - -// Build a content tree for each SDK -// Walk recursively through the learning-content dir, search for metadata files: -// content-info.yaml, module-info.yaml, unit-info.yaml. -func CollectLearningTree(rootpath string) (trees []tob.ContentTree, err error) { - err = filepath.WalkDir(rootpath, func(path string, d fs.DirEntry, err error) error { - // terminate walk on any error - if err != nil { - return err - } - if d.Name() == contentInfoYaml { - tree, err := collectSdk(path) - if err != nil { - return err - } - trees = append(trees, tree) - // don't walk into SDK subtree (already done by collectSdk) - return filepath.SkipDir - } - return nil - }) - - return trees, err -} diff --git a/learning/tour-of-beam/backend_v2/internal/fs_content/load_test.go b/learning/tour-of-beam/backend_v2/internal/fs_content/load_test.go deleted file mode 100644 index def3e9c1cd63..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/fs_content/load_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fs_content - -import ( - "path/filepath" - "testing" - - tob "beam.apache.org/learning/tour-of-beam/backend/internal" - "github.com/stretchr/testify/assert" -) - -func genUnitNode(id string) tob.Node { - return tob.Node{Type: tob.NODE_UNIT, Unit: &tob.Unit{ - Id: id, Name: "Challenge Name", - Description: "## Challenge description\n\nbla bla bla", - Hints: []string{ - "## Hint 1\n\napply yourself :)", - "## Hint 2\n\napply more", - }, - }} -} - -func TestSample(t *testing.T) { - trees, err := CollectLearningTree(filepath.Join("..", "..", "samples", "learning-content")) - assert.Nil(t, err) - assert.Equal(t, 2, len(trees)) - assert.Equal(t, tob.ContentTree{ - Sdk: tob.SDK_JAVA, - Modules: []tob.Module{ - { - Id: "module1", Name: "Module One", Complexity: "BASIC", - Nodes: []tob.Node{ - {Type: tob.NODE_UNIT, Unit: &tob.Unit{Id: "example1", Name: "Example Unit Name"}}, - genUnitNode("challenge1"), - }, - }, - { - Id: "module2", Name: "Module Two", Complexity: "MEDIUM", - Nodes: []tob.Node{ - {Type: tob.NODE_UNIT, Unit: &tob.Unit{Id: "example21", Name: "Example Unit Name"}}, - genUnitNode("challenge21"), - }, - }, - }, - }, trees[0]) - assert.Equal(t, tob.ContentTree{ - Sdk: tob.SDK_PYTHON, - Modules: []tob.Module{ - { - Id: "module1", Name: "Module One", Complexity: "BASIC", - Nodes: []tob.Node{ - {Type: tob.NODE_UNIT, Unit: &tob.Unit{Id: "intro-unit", Name: "Intro Unit Name"}}, - { - Type: tob.NODE_GROUP, Group: &tob.Group{ - Name: "The Group", Nodes: []tob.Node{ - {Type: tob.NODE_UNIT, Unit: &tob.Unit{Id: "example1", Name: "Example Unit Name"}}, - genUnitNode("challenge1"), - }, - }, - }, - }, - }, - }, - }, trees[1]) -} diff --git a/learning/tour-of-beam/backend_v2/internal/fs_content/yaml.go b/learning/tour-of-beam/backend_v2/internal/fs_content/yaml.go deleted file mode 100644 index 0546637a0d55..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/fs_content/yaml.go +++ /dev/null @@ -1,81 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fs_content - -import ( - "io/ioutil" - "log" - - "gopkg.in/yaml.v3" -) - -// Could have done it in generics if 1.18 was supported in GCF -// Fatals on error. -func loadLearningPathInfo(path string) (info learningPathInfo) { - buf, err := ioutil.ReadFile(path) - if err != nil { - log.Fatal(err) - } - - err = yaml.Unmarshal(buf, &info) - if err != nil { - log.Fatal(err) - } - - return info -} - -func loadLearningModuleInfo(path string) (info learningModuleInfo) { - buf, err := ioutil.ReadFile(path) - if err != nil { - log.Fatal(err) - } - - err = yaml.Unmarshal(buf, &info) - if err != nil { - log.Fatal(err) - } - - return info -} - -func loadLearningGroupInfo(path string) (info learningGroupInfo) { - buf, err := ioutil.ReadFile(path) - if err != nil { - log.Fatal(err) - } - - err = yaml.Unmarshal(buf, &info) - if err != nil { - log.Fatal(err) - } - - return info -} - -func loadLearningUnitInfo(path string) (info learningUnitInfo) { - buf, err := ioutil.ReadFile(path) - if err != nil { - log.Fatal(err) - } - - err = yaml.Unmarshal(buf, &info) - if err != nil { - log.Fatal(err) - } - - return info -} diff --git a/learning/tour-of-beam/backend_v2/internal/json.go b/learning/tour-of-beam/backend_v2/internal/json.go deleted file mode 100644 index 793f04b213c8..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/json.go +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -func (nt NodeType) MarshalText() ([]byte, error) { - var typ string - switch nt { - case NODE_UNIT: - typ = "unit" - case NODE_GROUP: - typ = "group" - default: - panic("NodeType not defined") - } - return []byte(typ), nil -} diff --git a/learning/tour-of-beam/backend_v2/internal/sdk.go b/learning/tour-of-beam/backend_v2/internal/sdk.go deleted file mode 100644 index a1451d183755..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/sdk.go +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -type Sdk string - -const ( - SDK_UNDEFINED Sdk = "" - SDK_GO Sdk = "Go" - SDK_PYTHON Sdk = "Python" - SDK_JAVA Sdk = "Java" - SDK_SCIO Sdk = "SCIO" -) - -func (s Sdk) String() string { - return string(s) -} - -// Parse sdk from string names, f.e. "Java" -> Sdk.GO_JAVA -// Returns SDK_UNDEFINED on error. -func ParseSdk(s string) Sdk { - switch s { - case "Go": - return SDK_GO - case "Python": - return SDK_PYTHON - case "Java": - return SDK_JAVA - case "SCIO": - return SDK_SCIO - default: - return SDK_UNDEFINED - } -} - -func SdksList() [4]string { - return [4]string{"Java", "Python", "Go", "SCIO"} -} diff --git a/learning/tour-of-beam/backend_v2/internal/sdk_test.go b/learning/tour-of-beam/backend_v2/internal/sdk_test.go deleted file mode 100644 index 562679952c1a..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/sdk_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import "testing" - -func TestParse(t *testing.T) { - for _, s := range []struct { - str string - expected Sdk - }{ - {"Go", SDK_GO}, - {"Python", SDK_PYTHON}, - {"Java", SDK_JAVA}, - {"SCIO", SDK_SCIO}, - {"Bad", SDK_UNDEFINED}, - {"", SDK_UNDEFINED}, - } { - if parsed := ParseSdk(s.str); parsed != s.expected { - t.Errorf("Failed to parse %v: got %v (expected %v)", s.str, parsed, s.expected) - } - } -} - -func TestSerialize(t *testing.T) { - for _, s := range []struct { - expected string - sdk Sdk - }{ - {"Go", SDK_GO}, - {"Python", SDK_PYTHON}, - {"Java", SDK_JAVA}, - {"SCIO", SDK_SCIO}, - {"", SDK_UNDEFINED}, - } { - if txt := s.sdk.String(); txt != s.expected { - t.Errorf("Failed to serialize %v to string: got %v (expected %v)", s.sdk, txt, s.expected) - } - } -} - -func TestSdkList(t *testing.T) { - if SdksList() != [4]string{"Java", "Python", "Go", "SCIO"} { - t.Error("Sdk list mismatch: ", SdksList()) - } -} diff --git a/learning/tour-of-beam/backend_v2/internal/service/content.go b/learning/tour-of-beam/backend_v2/internal/service/content.go deleted file mode 100644 index 2644e5a1a143..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/service/content.go +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package service - -import ( - "context" - - tob "beam.apache.org/learning/tour-of-beam/backend/internal" - "beam.apache.org/learning/tour-of-beam/backend/internal/storage" -) - -type IContent interface { - GetContentTree(ctx context.Context, sdk tob.Sdk, userId *string) (tob.ContentTree, error) - GetUnitContent(ctx context.Context, sdk tob.Sdk, unitId string, userId *string) (*tob.Unit, error) -} - -type Svc struct { - Repo storage.Iface -} - -func (s *Svc) GetContentTree(ctx context.Context, sdk tob.Sdk, userId *string) (ct tob.ContentTree, err error) { - // TODO enrich tree with user-specific state (isCompleted) - return s.Repo.GetContentTree(ctx, sdk) -} - -func (s *Svc) GetUnitContent(ctx context.Context, sdk tob.Sdk, unitId string, userId *string) (unit *tob.Unit, err error) { - // TODO enrich unit with user-specific state: isCompleted, userSnippetId - return s.Repo.GetUnitContent(ctx, sdk, unitId) -} diff --git a/learning/tour-of-beam/backend_v2/internal/service/mock.go b/learning/tour-of-beam/backend_v2/internal/service/mock.go deleted file mode 100644 index eee1c292b516..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/service/mock.go +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package service - -import ( - "context" - "encoding/json" - "io/ioutil" - "path" - "runtime" - - tob "beam.apache.org/learning/tour-of-beam/backend/internal" -) - -func getSamplesPath() string { - _, filepath, _, _ := runtime.Caller(1) - return path.Join(path.Dir(filepath), "..", "..", "samples", "api") -} - -type Mock struct{} - -// check if the interface is implemented. -var _ IContent = &Mock{} - -func (d *Mock) GetContentTree(_ context.Context, sdk tob.Sdk, userId *string) (ct tob.ContentTree, err error) { - content, _ := ioutil.ReadFile(path.Join(getSamplesPath(), "content_tree.json")) - _ = json.Unmarshal(content, &ct) - return ct, nil -} - -func (d *Mock) GetUnitContent(_ context.Context, sdk tob.Sdk, unitId string, userId *string) (u *tob.Unit, err error) { - content, _ := ioutil.ReadFile(path.Join(getSamplesPath(), "unit.json")) - err = json.Unmarshal(content, &u) - return u, err -} diff --git a/learning/tour-of-beam/backend_v2/internal/storage/adapter.go b/learning/tour-of-beam/backend_v2/internal/storage/adapter.go deleted file mode 100644 index ac7a1c2b25f3..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/storage/adapter.go +++ /dev/null @@ -1,146 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "fmt" - - tob "beam.apache.org/learning/tour-of-beam/backend/internal" - "cloud.google.com/go/datastore" -) - -func sdkToKey(sdk tob.Sdk) string { - switch sdk { - case tob.SDK_GO: - return "SDK_GO" - case tob.SDK_PYTHON: - return "SDK_PYTHON" - case tob.SDK_JAVA: - return "SDK_JAVA" - case tob.SDK_SCIO: - return "SDK_SCIO" - } - panic(fmt.Sprintf("Undefined key for sdk: %s", sdk)) -} - -func pgNameKey(kind, nameId string, parentKey *datastore.Key) (key *datastore.Key) { - key = datastore.NameKey(kind, nameId, parentKey) - key.Namespace = PgNamespace - return key -} - -// Get entity key from sdk & entity ID -// SDK_JAVA_{entityID}. -func datastoreKey(kind string, sdk tob.Sdk, id string, parent *datastore.Key) *datastore.Key { - name := fmt.Sprintf("%s_%s", sdkToKey(sdk), id) - return pgNameKey(kind, name, parent) -} - -func MakeUnitNode(unit *tob.Unit, order, level int) *TbLearningNode { - if unit == nil { - return nil - } - return &TbLearningNode{ - Id: unit.Id, - Name: unit.Name, - - Type: tob.NODE_UNIT, - Order: order, - Level: level, - - Unit: &TbLearningUnit{ - Id: unit.Id, - Name: unit.Name, - - Description: unit.Description, - Hints: unit.Hints, - TaskSnippetId: unit.TaskSnippetId, - SolutionSnippetId: unit.SolutionSnippetId, - }, - } -} - -func MakeGroupNode(group *tob.Group, order, level int) *TbLearningNode { - if group == nil { - return nil - } - return &TbLearningNode{ - // ID doesn't make much sense for groups, - // but we have to define it to include in queries - Id: group.Name, - Name: group.Name, - - Type: tob.NODE_GROUP, - Order: order, - Level: level, - - Group: &TbLearningGroup{ - Name: group.Name, - }, - } -} - -// Depending on the projection, we either convert TbLearningUnit to a model -// Or we use common fields Id, Name to make it. -func FromDatastoreUnit(tbUnit *TbLearningUnit, id, name string) *tob.Unit { - if tbUnit == nil { - return &tob.Unit{Id: id, Name: name} - } - return &tob.Unit{ - Id: tbUnit.Id, - Name: tbUnit.Name, - Description: tbUnit.Description, - Hints: tbUnit.Hints, - TaskSnippetId: tbUnit.TaskSnippetId, - SolutionSnippetId: tbUnit.SolutionSnippetId, - } -} - -// Depending on the projection, we either convert TbLearningGroup to a model -// Or we use common field Name to make it. -func FromDatastoreGroup(tbGroup *TbLearningGroup, name string) *tob.Group { - if tbGroup == nil { - return &tob.Group{Name: name} - } - return &tob.Group{ - Name: tbGroup.Name, - } -} - -func FromDatastoreNode(tbNode TbLearningNode) tob.Node { - node := tob.Node{ - Type: tbNode.Type, - } - switch tbNode.Type { - case tob.NODE_GROUP: - node.Group = FromDatastoreGroup(tbNode.Group, tbNode.Name) - case tob.NODE_UNIT: - node.Unit = FromDatastoreUnit(tbNode.Unit, tbNode.Id, tbNode.Name) - default: - panic("undefined node type") - } - return node -} - -func MakeDatastoreModule(mod *tob.Module, order int) *TbLearningModule { - return &TbLearningModule{ - Id: mod.Id, - Name: mod.Name, - Complexity: mod.Complexity, - - Order: order, - } -} diff --git a/learning/tour-of-beam/backend_v2/internal/storage/datastore.go b/learning/tour-of-beam/backend_v2/internal/storage/datastore.go deleted file mode 100644 index 62c55b2ba5b2..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/storage/datastore.go +++ /dev/null @@ -1,260 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "context" - "fmt" - "log" - - tob "beam.apache.org/learning/tour-of-beam/backend/internal" - "cloud.google.com/go/datastore" -) - -type DatastoreDb struct { - Client *datastore.Client -} - -// Query modules structure and content (recursively). -func (d *DatastoreDb) collectModules(ctx context.Context, tx *datastore.Transaction, - rootKey *datastore.Key, -) ([]tob.Module, error) { - // Custom index.yaml should be applied for this query to work - // (Ancestor + Order) - modules := make([]tob.Module, 0) - var tbMods []TbLearningModule - queryModules := datastore.NewQuery(TbLearningModuleKind). - Namespace(PgNamespace). - Ancestor(rootKey). - Order("order"). - Transaction(tx) - _, err := d.Client.GetAll(ctx, queryModules, &tbMods) - if err != nil { - return modules, fmt.Errorf("error querying modules for %v: %w", rootKey, err) - } - - for _, tbMod := range tbMods { - mod := tob.Module{Id: tbMod.Id, Name: tbMod.Name, Complexity: tbMod.Complexity} - mod.Nodes, err = d.collectNodes(ctx, tx, tbMod.Key, 0) - if err != nil { - return modules, err - } - modules = append(modules, mod) - } - return modules, nil -} - -// Get a group recursively. -// Params: -// - parentKey -// - level: depth of a node's children -// Recursively query/collect for each subgroup key, with level = level + 1. -func (d *DatastoreDb) collectNodes(ctx context.Context, tx *datastore.Transaction, - parentKey *datastore.Key, level int, -) (nodes []tob.Node, err error) { - var tbNodes []TbLearningNode - - // Custom index.yaml should be applied for this query to work - queryNodes := datastore.NewQuery(TbLearningNodeKind). - Namespace(PgNamespace). - Ancestor(parentKey). - FilterField("level", "=", level). - Project("type", "id", "name"). - Order("order"). - Transaction(tx) - if _, err = d.Client.GetAll(ctx, queryNodes, &tbNodes); err != nil { - return nodes, fmt.Errorf("getting children of node %v: %w", parentKey, err) - } - - // traverse the nodes which are groups, with level=level+1 - nodes = make([]tob.Node, 0, len(tbNodes)) - for _, tbNode := range tbNodes { - node := FromDatastoreNode(tbNode) - if node.Type == tob.NODE_GROUP { - node.Group.Nodes, err = d.collectNodes(ctx, tx, tbNode.Key, level+1) - } - if err != nil { - return nodes, err - } - nodes = append(nodes, node) - } - - return nodes, nil -} - -// Get learning content tree for SDK. -func (d *DatastoreDb) GetContentTree(ctx context.Context, sdk tob.Sdk) (tree tob.ContentTree, err error) { - var tbLP TbLearningPath - tree.Sdk = sdk - - _, err = d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error { - rootKey := pgNameKey(TbLearningPathKind, sdkToKey(sdk), nil) - if err := d.Client.Get(ctx, rootKey, &tbLP); err != nil { - return fmt.Errorf("error querying learning_path: %w", err) - } - tree.Modules, err = d.collectModules(ctx, tx, rootKey) - if err != nil { - return err - } - return nil - }, datastore.ReadOnly) - - return tree, err -} - -// Helper to clear all ToB Datastore entities related to a particular SDK -// They have one common ancestor key in tb_learning_path. -func (d *DatastoreDb) clearContentTree(ctx context.Context, tx *datastore.Transaction, sdk tob.Sdk) error { - rootKey := pgNameKey(TbLearningPathKind, sdkToKey(sdk), nil) - q := datastore.NewQuery(""). - Namespace(PgNamespace). - Ancestor(rootKey). - KeysOnly(). - Transaction(tx) - keys, err := d.Client.GetAll(ctx, q, nil) - if err != nil { - return err - } - - for _, key := range keys { - log.Println("deleting ", key) - } - - err = tx.DeleteMulti(keys) - if err != nil { - return err - } - return tx.Delete(rootKey) -} - -// Serialize a content tree to Datastore. -func (d *DatastoreDb) saveContentTree(tx *datastore.Transaction, tree *tob.ContentTree) error { - sdk := tree.Sdk - - saveUnit := func(unit *tob.Unit, order, level int, parentKey *datastore.Key) error { - unitKey := datastoreKey(TbLearningNodeKind, tree.Sdk, unit.Id, parentKey) - _, err := tx.Put(unitKey, MakeUnitNode(unit, order, level)) - if err != nil { - return fmt.Errorf("failed to put unit: %w", err) - } - return nil - } - - // transaction-wide autoincremented Id - // could have used numericID keys, if there was no transaction: - // incomplete keys are resolved after Tx commit, and - // we need to reference them in child nodes - var groupId int = 0 - genGroupKey := func(parentKey *datastore.Key) *datastore.Key { - groupId++ - return datastoreKey(TbLearningNodeKind, - tree.Sdk, fmt.Sprintf("group%v", groupId), parentKey) - } - - var saveNode func(tob.Node, int, int, *datastore.Key) error - saveGroup := func(group *tob.Group, order, level int, parentKey *datastore.Key) error { - groupKey := genGroupKey(parentKey) - if _, err := tx.Put(groupKey, MakeGroupNode(group, order, level)); err != nil { - return fmt.Errorf("failed to put group: %w", err) - } - for order, node := range group.Nodes { - if err := saveNode(node, order, level+1, groupKey); err != nil { - return err - } - } - return nil - } - - saveNode = func(node tob.Node, order, level int, parentKey *datastore.Key) error { - if node.Type == tob.NODE_UNIT { - return saveUnit(node.Unit, order, level, parentKey) - } else if node.Type == tob.NODE_GROUP { - return saveGroup(node.Group, order, level, parentKey) - } - - return fmt.Errorf("unknown datastore node type: %v", node.Type) - } - - rootKey := pgNameKey(TbLearningPathKind, sdkToKey(tree.Sdk), nil) - tbLP := TbLearningPath{Name: tree.Sdk.String()} - if _, err := tx.Put(rootKey, &tbLP); err != nil { - return fmt.Errorf("failed to put learning_path: %w", err) - } - - for order, mod := range tree.Modules { - modKey := datastoreKey(TbLearningModuleKind, sdk, mod.Id, rootKey) - if _, err := tx.Put(modKey, MakeDatastoreModule(&mod, order)); err != nil { - return fmt.Errorf("failed to put module: %w", err) - } - for order, node := range mod.Nodes { - if err := saveNode(node, order /*module node is at level 0*/, 0, modKey); err != nil { - return err - } - } - } - - return nil -} - -// Re-create content trees for each SDK in separate transaction. -func (d *DatastoreDb) SaveContentTrees(ctx context.Context, trees []tob.ContentTree) error { - for _, tree := range trees { - log.Println("Saving sdk tree", tree.Sdk) - _, err := d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error { - if err := d.clearContentTree(ctx, tx, tree.Sdk); err != nil { - return err - } - return d.saveContentTree(tx, &tree) - }) - if err != nil { - return err - } - } - - return nil -} - -// Get learning unit content by unitId -func (d *DatastoreDb) GetUnitContent(ctx context.Context, sdk tob.Sdk, unitId string) (unit *tob.Unit, err error) { - var tbNodes []TbLearningNode - rootKey := pgNameKey(TbLearningPathKind, sdkToKey(sdk), nil) - - query := datastore.NewQuery(TbLearningNodeKind). - Namespace(PgNamespace). - Ancestor(rootKey). - FilterField("id", "=", unitId) - - _, err = d.Client.GetAll(ctx, query, &tbNodes) - if err != nil { - return nil, fmt.Errorf("query unit failed: %w", err) - } - - switch { - case len(tbNodes) == 0: - return nil, nil - case len(tbNodes) > 1: - return nil, fmt.Errorf("query by unitId returned %v units", len(tbNodes)) - } - - node := FromDatastoreNode(tbNodes[0]) - if node.Type != tob.NODE_UNIT { - return nil, fmt.Errorf("wrong node type: %v, unit expected", node.Type) - } - return node.Unit, nil -} - -// check if the interface is implemented. -var _ Iface = &DatastoreDb{} diff --git a/learning/tour-of-beam/backend_v2/internal/storage/iface.go b/learning/tour-of-beam/backend_v2/internal/storage/iface.go deleted file mode 100644 index f81a28e4ce52..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/storage/iface.go +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "context" - - tob "beam.apache.org/learning/tour-of-beam/backend/internal" -) - -type Iface interface { - GetContentTree(ctx context.Context, sdk tob.Sdk) (tob.ContentTree, error) - SaveContentTrees(ctx context.Context, trees []tob.ContentTree) error - - GetUnitContent(ctx context.Context, sdk tob.Sdk, unitId string) (*tob.Unit, error) -} diff --git a/learning/tour-of-beam/backend_v2/internal/storage/image/Dockerfile b/learning/tour-of-beam/backend_v2/internal/storage/image/Dockerfile deleted file mode 100644 index f8f5a4b92b8c..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/storage/image/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Version. Can change in build progress -ARG GCLOUD_SDK_VERSION=397.0.0-emulators - -# Use google cloud sdk -FROM google/cloud-sdk:$GCLOUD_SDK_VERSION - -# Volume to persist Datastore data -VOLUME /opt/data - -# RUN mkdir -p /opt/data/WEB-INF -# COPY index.yaml /opt/data/WEB-INF/index.yaml -COPY start-datastore.sh . - -EXPOSE 8081 - -ENTRYPOINT ["./start-datastore.sh"] diff --git a/learning/tour-of-beam/backend_v2/internal/storage/image/start-datastore.sh b/learning/tour-of-beam/backend_v2/internal/storage/image/start-datastore.sh deleted file mode 100644 index bf0ccf3f7eba..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/storage/image/start-datastore.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Check user environment variable -if [[ -z "${DATASTORE_PROJECT_ID}" ]]; then - echo "Missing DATASTORE_PROJECT_ID environment variable" >&2 - exit 1 -fi - -if [[ -z "${DATASTORE_LISTEN_ADDRESS}" ]]; then - echo "Missing DATASTORE_LISTEN_ADDRESS environment variable" >&2 - exit 1 -fi - -options=${options:1} -# Check for datastore options -while [ ! $# -eq 0 ] -do - case "$1" in - --store-on-disk) - options="$options --store-on-disk" - shift - ;; - --no-store-on-disk) - options="$options --no-store-on-disk" - shift - ;; - --consistency=*) - consistency=${1#*=} - options="$options --consistency=$consistency" - shift - ;; - *) - echo "Invalid option: $1. Use: --store-on-disk, --no-store-on-disk, --consistency=[0.0-1.0]" - exit 1 - ;; - esac -done - -# Config gcloud project -gcloud config set project ${DATASTORE_PROJECT_ID} - -# Start emulator -gcloud beta emulators datastore start \ - --data-dir=/opt/data \ - --host-port=${DATASTORE_LISTEN_ADDRESS} \ - ${options} diff --git a/learning/tour-of-beam/backend_v2/internal/storage/index.yaml b/learning/tour-of-beam/backend_v2/internal/storage/index.yaml deleted file mode 100644 index fa78d72f9481..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/storage/index.yaml +++ /dev/null @@ -1,20 +0,0 @@ -indexes: -# AUTOGENERATED - -# This index.yaml is automatically updated whenever the Cloud Datastore -# emulator detects that a new type of query is run. If you want to manage the -# index.yaml file manually, remove the "# AUTOGENERATED" marker line above. -# If you want to manage some indexes manually, move them above the marker line. - -- kind: "tb_learning_module" - ancestor: yes - properties: - - name: "order" -- kind: "tb_learning_node" - ancestor: yes - properties: - - name: "level" - - name: "order" - - name: "id" - - name: "name" - - name: "type" diff --git a/learning/tour-of-beam/backend_v2/internal/storage/schema.go b/learning/tour-of-beam/backend_v2/internal/storage/schema.go deleted file mode 100644 index 4e2b31679203..000000000000 --- a/learning/tour-of-beam/backend_v2/internal/storage/schema.go +++ /dev/null @@ -1,99 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - tob "beam.apache.org/learning/tour-of-beam/backend/internal" - "cloud.google.com/go/datastore" -) - -// ToB Datastore schema -// - Content tree has a root entity in tb_learning_path, -// descendant entities in tb_learning_module/group/unit have it as a common ancestor -// - learning path consists of modules, modules consist of groups and units -// - Ordering is established by "order" property -// - To limit ancestor queries by only first-level descendants, "level" property is used - -const ( - PgNamespace = "Tour" - - TbLearningPathKind = "tb_learning_path" - TbLearningModuleKind = "tb_learning_module" - TbLearningNodeKind = "tb_learning_node" - - PgSnippetsKind = "pg_snippets" - PgSdksKind = "pg_sdks" -) - -// tb_learning_path. -type TbLearningPath struct { - Key *datastore.Key `datastore:"__key__"` - Name string `datastore:"name"` -} - -// tb_learning_module. -type TbLearningModule struct { - Key *datastore.Key `datastore:"__key__"` - Id string `datastore:"id"` - Name string `datastore:"name"` - Complexity string `datastore:"complexity"` - - // internal, only db - Order int `datastore:"order"` -} - -// tb_learning_node.group. -type TbLearningGroup struct { - Name string `datastore:"name"` -} - -// tb_learning_node.unit -// Learning Unit content. -type TbLearningUnit struct { - Id string `datastore:"id"` - Name string `datastore:"name"` - Description string `datastore:"description,noindex"` - Hints []string `datastore:"hints,noindex"` - - TaskSnippetId string `datastore:"taskSnippetId"` - SolutionSnippetId string `datastore:"solutionSnippetId"` -} - -// tb_learning_node -// Container for learning tree nodes, which are either groups or units. -type TbLearningNode struct { - Type tob.NodeType `datastore:"type"` - // common fields, duplicate same fields from the nested entities - // (needed to allow projection when getting the content tree) - Id string `datastore:"id"` - Name string `datastore:"name"` - - // type-specific nested info - Unit *TbLearningUnit `datastore:"unit,noindex"` - Group *TbLearningGroup `datastore:"group,noindex"` - - // internal datastore-only fields - // key: _ - Key *datastore.Key `datastore:"__key__"` - Order int `datastore:"order"` - Level int `datastore:"level"` -} - -type PgSnippets struct { - Key *datastore.Key `datastore:"__key__"` - Origin string `datastore:"origin"` - Sdk *datastore.Key `datastore:"sdk"` -} diff --git a/learning/tour-of-beam/backend_v2/samples/api/get_content_tree.json b/learning/tour-of-beam/backend_v2/samples/api/get_content_tree.json deleted file mode 100644 index cf8c40b6f73b..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/api/get_content_tree.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "modules" : [ - { - "complexity" : "BASIC", - "moduleId" : "module1", - "name" : "Module One", - "nodes" : [ - { - "type" : "unit", - "unit" : { - "name" : "Intro Unit Name", - "unitId" : "intro-unit" - } - }, - { - "group" : { - "name" : "The Group", - "nodes" : [ - { - "type" : "unit", - "unit" : { - "name" : "Example Unit Name", - "unitId" : "example1" - } - }, - { - "type" : "unit", - "unit" : { - "name" : "Challenge Name", - "unitId" : "challenge1" - } - } - ] - }, - "type" : "group" - } - ] - } - ], - "sdk" : "Python" -} \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/api/get_unit_content.json b/learning/tour-of-beam/backend_v2/samples/api/get_unit_content.json deleted file mode 100644 index a18305870ceb..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/api/get_unit_content.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "unitId": "1.1", - "name": "Basic concepts", - "description": "Lorem ipsum...", - "hint": "Try to use this method....", - "assignment": "assignmentSnippetId", - "solution": "solutionSnippetId", - "userSnippet": "userSnippetId", - "completed": true -} diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/content-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/content-info.yaml deleted file mode 100644 index 0d6a17105315..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/content-info.yaml +++ /dev/null @@ -1,4 +0,0 @@ -sdk: Java -content: - - module 1 - - module 2 \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/module-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/module-info.yaml deleted file mode 100644 index 04384cb56b31..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/module-info.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: module1 -name: Module One -complexity: BASIC -content: -- unit-example -- unit-challenge - diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/description.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/description.md deleted file mode 100644 index 1906064ed20f..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/description.md +++ /dev/null @@ -1,3 +0,0 @@ -## Challenge description - -bla bla bla \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint1.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint1.md deleted file mode 100644 index e5d1a67fa906..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint1.md +++ /dev/null @@ -1,3 +0,0 @@ -## Hint 1 - -apply yourself :) \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint2.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint2.md deleted file mode 100644 index 0c425751be9f..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/hint2.md +++ /dev/null @@ -1,3 +0,0 @@ -## Hint 2 - -apply more \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml deleted file mode 100644 index e20b873a3e05..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-challenge/unit-info.yaml +++ /dev/null @@ -1,5 +0,0 @@ -id: challenge1 -name: Challenge Name -complexity: BASIC -taskName: ChallengeTask -solutionName: ChallengeSolution \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-example/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-example/unit-info.yaml deleted file mode 100644 index ed0c81854cb0..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 1/unit-example/unit-info.yaml +++ /dev/null @@ -1,3 +0,0 @@ -id: example1 -name: Example Unit Name -taskName: ExampleName \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/module-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/module-info.yaml deleted file mode 100644 index e152c45d22e1..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/module-info.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: module2 -name: Module Two -complexity: MEDIUM -content: -- unit-example -- unit-challenge - diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/description.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/description.md deleted file mode 100644 index 1906064ed20f..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/description.md +++ /dev/null @@ -1,3 +0,0 @@ -## Challenge description - -bla bla bla \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint1.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint1.md deleted file mode 100644 index e5d1a67fa906..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint1.md +++ /dev/null @@ -1,3 +0,0 @@ -## Hint 1 - -apply yourself :) \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint2.md b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint2.md deleted file mode 100644 index 0c425751be9f..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/hint2.md +++ /dev/null @@ -1,3 +0,0 @@ -## Hint 2 - -apply more \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml deleted file mode 100644 index ac65e8357d4c..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-challenge/unit-info.yaml +++ /dev/null @@ -1,5 +0,0 @@ -id: challenge21 -name: Challenge Name -complexity: BASIC -taskName: ChallengeTask -solutionName: ChallengeSolution \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-example/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-example/unit-info.yaml deleted file mode 100644 index 674a6da71941..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/java/module 2/unit-example/unit-info.yaml +++ /dev/null @@ -1,3 +0,0 @@ -id: example21 -name: Example Unit Name -taskName: ExampleName \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/content-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/content-info.yaml deleted file mode 100644 index ee9d329b5030..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/python/content-info.yaml +++ /dev/null @@ -1,3 +0,0 @@ -sdk: Python -content: - - module 1 diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/group-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/group-info.yaml deleted file mode 100644 index 1619bc19fb91..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/group-info.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: group -name: The Group -complexity: BASIC -content: -- unit-example -- unit-challenge - diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/description.md b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/description.md deleted file mode 100644 index 1906064ed20f..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/description.md +++ /dev/null @@ -1,3 +0,0 @@ -## Challenge description - -bla bla bla \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint1.md b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint1.md deleted file mode 100644 index e5d1a67fa906..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint1.md +++ /dev/null @@ -1,3 +0,0 @@ -## Hint 1 - -apply yourself :) \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint2.md b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint2.md deleted file mode 100644 index 0c425751be9f..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/hint2.md +++ /dev/null @@ -1,3 +0,0 @@ -## Hint 2 - -apply more \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml deleted file mode 100644 index e20b873a3e05..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-challenge/unit-info.yaml +++ /dev/null @@ -1,5 +0,0 @@ -id: challenge1 -name: Challenge Name -complexity: BASIC -taskName: ChallengeTask -solutionName: ChallengeSolution \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml deleted file mode 100644 index ed0c81854cb0..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/group/unit-example/unit-info.yaml +++ /dev/null @@ -1,3 +0,0 @@ -id: example1 -name: Example Unit Name -taskName: ExampleName \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/intro-unit/unit-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/intro-unit/unit-info.yaml deleted file mode 100644 index 1bcdb8d1abd1..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/intro-unit/unit-info.yaml +++ /dev/null @@ -1,3 +0,0 @@ -id: intro-unit -name: Intro Unit Name -taskName: IntroTask \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/module-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/module-info.yaml deleted file mode 100644 index 024d12fdc69c..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content/python/module 1/module-info.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: module1 -name: Module One -complexity: BASIC -content: -- intro-unit -- group - diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content_empty/java/content-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content_empty/java/content-info.yaml deleted file mode 100644 index 66ecd69e089d..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content_empty/java/content-info.yaml +++ /dev/null @@ -1,2 +0,0 @@ -sdk: Java -content: [] \ No newline at end of file diff --git a/learning/tour-of-beam/backend_v2/samples/learning-content_empty/python/content-info.yaml b/learning/tour-of-beam/backend_v2/samples/learning-content_empty/python/content-info.yaml deleted file mode 100644 index cf165934c106..000000000000 --- a/learning/tour-of-beam/backend_v2/samples/learning-content_empty/python/content-info.yaml +++ /dev/null @@ -1,2 +0,0 @@ -sdk: Python -content: [] diff --git a/learning/tour-of-beam/backend_v2/test.txt b/learning/tour-of-beam/backend_v2/test.txt deleted file mode 100644 index 08c9db954f6e..000000000000 --- a/learning/tour-of-beam/backend_v2/test.txt +++ /dev/null @@ -1 +0,0 @@ -Hello There! \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/README.md b/learning/tour-of-beam/terraform-v2/README.md new file mode 100644 index 000000000000..72ffa5faf7c6 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/README.md @@ -0,0 +1,56 @@ +# [Tour of Beam] Deploying backend infrastructure as GCP Cloud Function with GCP, Terraform and Cloud Build +This tutorial explains how to create Tour of Beam backend infrastructure as a GCP Cloud Functions using Infrastructure as a Code (IaC) and GitOps methodoliges. + + +### Deployment +Prerequisites: +- Active GCP project with billing enabled +- Existing bucket to store Terraform state (name to be declared in backend.tf) +- Existing service account with the following roles: + - Cloud Functions Admin + - Project IAM Admin +- Exported JSON Key of service account above + +#### Configuring your environment + +Steps below will: +1. Enable required APIs for the project +2. Create service account and assign required IAM roles to it (service account to run the function with) +3. Create bucket to archive and store source code +4. Create cloud functions to each function defined in backend source code + + +```bash +# Create environment directory per your requirements/policy +mkdir environments/dev +cd ../environments/dev +# Import exported JSON Key using +export GOOGLE_APPLICATION_CREDENTIALS = '..key path...' +# Initiliaze and run terraform +terraform init +terraform plan +terraform apply +terraform destroy +``` + + +### Sample usage + +Entry point: list sdk names +``` +$ curl -X GET https://$REGION-$PROJECT_ID.cloudfunctions.net/getSdkList | json_pp +``` +[response](./samples/api/get_sdk_list.json) + +Get content tree by sdk name (SDK name == SDK id) +``` +$ curl -X GET 'https://$REGION-$PROJECT_ID.cloudfunctions.net/getContentTree?sdk=python' +``` +[response](./samples/api/get_content_tree.json) + + +Get unit content tree by sdk name and unitId +``` +$ curl -X GET 'https://$REGION-$PROJECT_ID.cloudfunctions.net/getContentTree?sdk=python&id=challenge1' +``` +[response](./samples/api/get_unit_content.json) \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf b/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf index 007249c9120c..97eb4336865c 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf @@ -2,16 +2,16 @@ output "service-account-email" { value = module.iam.service-account-email } -output "cloud-function-trigger-url" { - value = module.cloud_functions.cloud-function-trigger-url +output "cloud-function-trigger-url1" { + value = module.cloud_functions.cloud-function-trigger-url1 } -output "cloud-function-name" { - value = module.cloud_functions.cloud-function-name +output "cloud-function-trigger-url2" { + value = module.cloud_functions.cloud-function-trigger-url2 } -output "functions-bucket-id" { - value = module.buckets.functions-bucket-id +output "cloud-function-trigger-url3" { + value = module.cloud_functions.cloud-function-trigger-url3 } output "functions-bucket-name" { diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 1b765ee7bac0..3eba82b2bc35 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -1,5 +1,5 @@ -resource "google_cloudfunctions_function" "cloud_function" { - name = "tour-of-beam-backend-cloud-function" +resource "google_cloudfunctions_function" "cloud_function1" { + name = "tour-of-beam-backend-cloud-function1" runtime = "go116" project = var.project_id service_account_email = var.service_account_id @@ -19,12 +19,80 @@ resource "google_cloudfunctions_function" "cloud_function" { } +resource "google_cloudfunctions_function" "cloud_function2" { + name = "tour-of-beam-backend-cloud-function2" + runtime = "go116" + project = var.project_id + service_account_email = var.service_account_id + source_archive_bucket = var.source_archive_bucket + source_archive_object = var.source_archive_object + region = var.region + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered + entry_point = "getContentTree" + + environment_variables = { + DATASTORE_PROJECT_ID="test-cloud-func-deploy" + TOB_MOCK=1 + } + +} + +resource "google_cloudfunctions_function" "cloud_function3" { + name = "tour-of-beam-backend-cloud-function3" + runtime = "go116" + project = var.project_id + service_account_email = var.service_account_id + source_archive_bucket = var.source_archive_bucket + source_archive_object = var.source_archive_object + region = var.region + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered + entry_point = "getUnitContent" + + environment_variables = { + DATASTORE_PROJECT_ID="test-cloud-func-deploy" + TOB_MOCK=1 + } + +} + # Create IAM entry so all users can invoke the function -resource "google_cloudfunctions_function_iam_member" "invoker" { - project = google_cloudfunctions_function.cloud_function.project - region = google_cloudfunctions_function.cloud_function.region - cloud_function = google_cloudfunctions_function.cloud_function.name +resource "google_cloudfunctions_function_iam_member" "invoker1" { + project = google_cloudfunctions_function.cloud_function1.project + region = google_cloudfunctions_function.cloud_function1.region + cloud_function = google_cloudfunctions_function.cloud_function1.name role = "roles/cloudfunctions.invoker" member = "allUsers" + + depends_on = [google_cloudfunctions_function.cloud_function1] +} + +# Create IAM entry so all users can invoke the function +resource "google_cloudfunctions_function_iam_member" "invoker2" { + project = google_cloudfunctions_function.cloud_function2.project + region = google_cloudfunctions_function.cloud_function2.region + cloud_function = google_cloudfunctions_function.cloud_function2.name + + role = "roles/cloudfunctions.invoker" + member = "allUsers" + + depends_on = [google_cloudfunctions_function.cloud_function2] +} + +# Create IAM entry so all users can invoke the function +resource "google_cloudfunctions_function_iam_member" "invoker3" { + project = google_cloudfunctions_function.cloud_function3.project + region = google_cloudfunctions_function.cloud_function3.region + cloud_function = google_cloudfunctions_function.cloud_function3.name + + role = "roles/cloudfunctions.invoker" + member = "allUsers" + + depends_on = [google_cloudfunctions_function.cloud_function3] } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf index 227787919121..affa9bbb6fd2 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf @@ -1,7 +1,11 @@ -output "cloud-function-trigger-url" { - value = google_cloudfunctions_function.cloud_function.https_trigger_url +output "cloud-function-trigger-url1" { + value = google_cloudfunctions_function.cloud_function1.https_trigger_url } -output "cloud-function-name" { - value = google_cloudfunctions_function.cloud_function.name +output "cloud-function-trigger-url2" { + value = google_cloudfunctions_function.cloud_function2.https_trigger_url +} + +output "cloud-function-trigger-url3" { + value = google_cloudfunctions_function.cloud_function3.https_trigger_url } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index d91e46e16b0d..e5a0e62027aa 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -6,7 +6,7 @@ resource "google_service_account" "tourofbeam_cf_sa" { resource "google_project_iam_member" "terraform_service_account_roles" { for_each = toset([ "roles/resourcemanager.projectIamAdmin", "roles/cloudfunctions.admin", "roles/storage.objectViewer", - "roles/storage.objectCreator", "roles/iam.serviceAccountAdmin", + "roles/storage.objectCreator", "roles/iam.serviceAccountUser" ]) role = each.key member = "serviceAccount:${google_service_account.tourofbeam_cf_sa.email}" From 6e7c37ef3d5711f4d68d2e2bf6915db12f796f87 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Sun, 16 Oct 2022 13:49:47 -0700 Subject: [PATCH 175/269] Final update --- learning/tour-of-beam/terraform-v2/README.md | 4 + .../terraform-v2/environments/uat/outputs.tf | 12 +-- .../modules/cloud_functions/main.tf | 84 +++---------------- .../modules/cloud_functions/outputs.tf | 12 +-- .../modules/cloud_functions/variables.tf | 6 +- 5 files changed, 23 insertions(+), 95 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/README.md b/learning/tour-of-beam/terraform-v2/README.md index 72ffa5faf7c6..17ac6b6eb7f8 100644 --- a/learning/tour-of-beam/terraform-v2/README.md +++ b/learning/tour-of-beam/terraform-v2/README.md @@ -9,6 +9,10 @@ Prerequisites: - Existing service account with the following roles: - Cloud Functions Admin - Project IAM Admin + - Service Account Admin + - Service Account User + - Storage Admin + - Storage Object Admin - Exported JSON Key of service account above #### Configuring your environment diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf b/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf index 97eb4336865c..ca0b2b553155 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf @@ -2,16 +2,8 @@ output "service-account-email" { value = module.iam.service-account-email } -output "cloud-function-trigger-url1" { - value = module.cloud_functions.cloud-function-trigger-url1 -} - -output "cloud-function-trigger-url2" { - value = module.cloud_functions.cloud-function-trigger-url2 -} - -output "cloud-function-trigger-url3" { - value = module.cloud_functions.cloud-function-trigger-url3 +output "cloud-function-trigger-url" { + value = module.cloud_functions.cloud-function-trigger-url } output "functions-bucket-name" { diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 3eba82b2bc35..866e1f5a33fa 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -1,5 +1,6 @@ -resource "google_cloudfunctions_function" "cloud_function1" { - name = "tour-of-beam-backend-cloud-function1" +resource "google_cloudfunctions_function" "cloud_function" { + count = length(var.entry_point_names) + name = "tour-of-beam-backend-cloud-function-${count.index}" runtime = "go116" project = var.project_id service_account_email = var.service_account_id @@ -10,7 +11,7 @@ resource "google_cloudfunctions_function" "cloud_function1" { # Get the source code of the cloud function as a Zip compression trigger_http = true # Name of the function that will be executed when the Google Cloud Function is triggered - entry_point = "getSdkList" + entry_point = var.entry_point_names[count.index] environment_variables = { DATASTORE_PROJECT_ID="test-cloud-func-deploy" @@ -19,80 +20,15 @@ resource "google_cloudfunctions_function" "cloud_function1" { } -resource "google_cloudfunctions_function" "cloud_function2" { - name = "tour-of-beam-backend-cloud-function2" - runtime = "go116" - project = var.project_id - service_account_email = var.service_account_id - source_archive_bucket = var.source_archive_bucket - source_archive_object = var.source_archive_object - region = var.region - ingress_settings = "ALLOW_ALL" - # Get the source code of the cloud function as a Zip compression - trigger_http = true - # Name of the function that will be executed when the Google Cloud Function is triggered - entry_point = "getContentTree" - - environment_variables = { - DATASTORE_PROJECT_ID="test-cloud-func-deploy" - TOB_MOCK=1 - } - -} - -resource "google_cloudfunctions_function" "cloud_function3" { - name = "tour-of-beam-backend-cloud-function3" - runtime = "go116" - project = var.project_id - service_account_email = var.service_account_id - source_archive_bucket = var.source_archive_bucket - source_archive_object = var.source_archive_object - region = var.region - ingress_settings = "ALLOW_ALL" - # Get the source code of the cloud function as a Zip compression - trigger_http = true - # Name of the function that will be executed when the Google Cloud Function is triggered - entry_point = "getUnitContent" - - environment_variables = { - DATASTORE_PROJECT_ID="test-cloud-func-deploy" - TOB_MOCK=1 - } - -} - -# Create IAM entry so all users can invoke the function -resource "google_cloudfunctions_function_iam_member" "invoker1" { - project = google_cloudfunctions_function.cloud_function1.project - region = google_cloudfunctions_function.cloud_function1.region - cloud_function = google_cloudfunctions_function.cloud_function1.name - - role = "roles/cloudfunctions.invoker" - member = "allUsers" - - depends_on = [google_cloudfunctions_function.cloud_function1] -} - -# Create IAM entry so all users can invoke the function -resource "google_cloudfunctions_function_iam_member" "invoker2" { - project = google_cloudfunctions_function.cloud_function2.project - region = google_cloudfunctions_function.cloud_function2.region - cloud_function = google_cloudfunctions_function.cloud_function2.name - - role = "roles/cloudfunctions.invoker" - member = "allUsers" - - depends_on = [google_cloudfunctions_function.cloud_function2] -} - # Create IAM entry so all users can invoke the function -resource "google_cloudfunctions_function_iam_member" "invoker3" { - project = google_cloudfunctions_function.cloud_function3.project - region = google_cloudfunctions_function.cloud_function3.region - cloud_function = google_cloudfunctions_function.cloud_function3.name +resource "google_cloudfunctions_function_iam_member" "invoker" { + count = length(google_cloudfunctions_function.cloud_function) + project = google_cloudfunctions_function.cloud_function[count.index].project + region = google_cloudfunctions_function.cloud_function[count.index].region + cloud_function = google_cloudfunctions_function.cloud_function[count.index].name role = "roles/cloudfunctions.invoker" member = "allUsers" - depends_on = [google_cloudfunctions_function.cloud_function3] + depends_on = [google_cloudfunctions_function.cloud_function] } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf index affa9bbb6fd2..72b116a897e8 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf @@ -1,11 +1,3 @@ -output "cloud-function-trigger-url1" { - value = google_cloudfunctions_function.cloud_function1.https_trigger_url -} - -output "cloud-function-trigger-url2" { - value = google_cloudfunctions_function.cloud_function2.https_trigger_url -} - -output "cloud-function-trigger-url3" { - value = google_cloudfunctions_function.cloud_function3.https_trigger_url +output "cloud-function-trigger-url" { + value = google_cloudfunctions_function.cloud_function.*.https_trigger_url } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf index 4cbec6697f73..189b192e08ef 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -5,5 +5,9 @@ variable "service_account_id" { } variable "bucket_name" {} variable "source_archive_bucket" {} +variable "source_archive_object" {} -variable "source_archive_object" {} \ No newline at end of file +variable "entry_point_names" { + type = list + default = ["getSdkList", "getContentTree", "getUnitContent"] +} \ No newline at end of file From 0eb5c6f3383330127fc4307e71db1d5cdcbfc9a0 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Sun, 16 Oct 2022 13:54:03 -0700 Subject: [PATCH 176/269] Delete redundant dir cloud-build --- cloud-build/.dockerignore | 4 - cloud-build/.github/workflows/cloud-build.yml | 63 --- cloud-build/Dockerfile | 34 -- cloud-build/README.md | 88 ----- cloud-build/index.js | 28 -- cloud-build/package-lock.json | 374 ------------------ cloud-build/package.json | 14 - 7 files changed, 605 deletions(-) delete mode 100644 cloud-build/.dockerignore delete mode 100644 cloud-build/.github/workflows/cloud-build.yml delete mode 100644 cloud-build/Dockerfile delete mode 100644 cloud-build/README.md delete mode 100644 cloud-build/index.js delete mode 100644 cloud-build/package-lock.json delete mode 100644 cloud-build/package.json diff --git a/cloud-build/.dockerignore b/cloud-build/.dockerignore deleted file mode 100644 index 07efc8ee2472..000000000000 --- a/cloud-build/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -Dockerfile -README.md -node_modules -npm-debug.log diff --git a/cloud-build/.github/workflows/cloud-build.yml b/cloud-build/.github/workflows/cloud-build.yml deleted file mode 100644 index 6d9778acd893..000000000000 --- a/cloud-build/.github/workflows/cloud-build.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2019 Google, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Build using Cloud Build - -on: - push: - branches: - - 'main' - -env: - PROJECT_ID: ${{ secrets.RUN_PROJECT }} - SERVICE_NAME: helloworld-nodejs - -jobs: - setup-build-deploy: - name: Setup, Build, and Deploy - runs-on: ubuntu-latest - - # Add "id-token" with the intended permissions. - permissions: - contents: 'read' - id-token: 'write' - - steps: - - name: Checkout - uses: actions/checkout@v3 - - # Configure Workload Identity Federation and generate an access token. - - id: 'auth' - name: 'Authenticate to Google Cloud' - uses: 'google-github-actions/auth@v0' - with: - workload_identity_provider: 'projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider' - service_account: '${{ secrets.RUN_SA_EMAIL }}' - - # Alternative option - authentication via credentials json - # - id: 'auth' - # uses: 'google-github-actions/auth@v0' - # with: - # credentials_json: '${{ secrets.RUN_SA_KEY }}' - - # Setup gcloud CLI - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - - # Build and push image to Google Container Registry - - name: Build - run: |- - gcloud builds submit \ - --quiet \ - --tag "gcr.io/$PROJECT_ID/$SERVICE_NAME:$GITHUB_SHA" diff --git a/cloud-build/Dockerfile b/cloud-build/Dockerfile deleted file mode 100644 index 4eee05e71058..000000000000 --- a/cloud-build/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2020 Google, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Use the official lightweight Node.js 16 image. -# https://hub.docker.com/_/node -FROM node:16-slim - -# Create and change to the app directory. -WORKDIR /usr/src/app - -# Copy application dependency manifests to the container image. -# A wildcard is used to ensure both package.json AND package-lock.json are copied. -# Copying this separately prevents re-running npm install on every code change. -COPY package*.json ./ - -# Install production dependencies. -RUN npm install --only=production - -# Copy local code to the container image. -COPY . ./ - -# Run the web service on container startup. -CMD [ "npm", "start" ] diff --git a/cloud-build/README.md b/cloud-build/README.md deleted file mode 100644 index 5dad3b85b56c..000000000000 --- a/cloud-build/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# Cloud Build - GitHub Actions - -An example workflow that uses [GitHub Actions][actions] to build a -[Hello World Node.js app](index.js) container image using [Cloud Build][cloud-build]. - -This code is intended to be an _example_. You will likely need to change or -update values to match your setup. - -## Workflow description - -For pushes to the `main` branch, this workflow will: - -1. Download and configure the Google [Cloud SDK][sdk] with the provided - credentials. - -1. Build, tag, and push a container image to Google Container Registry. - - - The image is built using Cloud Build and pushed to Google Container Registry. - - - The image is available through the following tags: `latest` and first 8 of - the commit SHA. - -## Setup - -1. Create a new Google Cloud Project (or select an existing project) and - [enable the Cloud Build and Cloud Build APIs](https://console.cloud.google.com/flows/enableapi?apiid=cloudbuild.googleapis.com,run.googleapis.com). - -1. Create or reuse a GitHub repository for the example workflow: - - 1. [Create a repository](https://help.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-new-repository). - - 1. Move into the repository directory: - - ``` - $ cd - ``` - - 1. Copy the example into the repository: - - ``` - $ cp -r /github-actions/example-workflows/cloud-build/ . - ``` - -1. [Create a Google Cloud service account][create-sa] if one does not already - exist. - -1. Add the following [Cloud IAM roles][roles] to your service account: - - - [`Cloud Build Service Account`](https://cloud.google.com/cloud-build/docs/cloud-build-service-account) - allows for execution of builds on your behalf - - - `Viewer` - allows for Cloud Build log storage - -1. [Create a JSON service account key][create-key] for the service account. - -1. Add the following secrets to your repository's secrets: - - - `RUN_PROJECT`: Google Cloud project ID - - - `RUN_SA_EMAIL`: the email of the service account - - - `RUN_SA_KEY`: the content of the service account JSON file - -## Run the workflow - -1. Add and commit your changes: - - ```text - $ git add . - $ git commit -m "Set up GitHub workflow" - ``` - -1. Push to the `main` branch: - - ```text - $ git push -u origin main - ``` - -1. View the GitHub Actions Workflow by selecting the `Actions` tab at the top - of your repository on GitHub. Then click on the `Build using Cloud Build` - element to see the details. - -[actions]: https://help.github.com/en/categories/automating-your-workflow-with-github-actions -[cloud-build]: https://cloud.google.com/cloud-build/ -[create-sa]: https://cloud.google.com/iam/docs/creating-managing-service-accounts -[create-key]: https://cloud.google.com/iam/docs/creating-managing-service-account-keys -[sdk]: https://cloud.google.com/sdk -[secrets]: https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets -[roles]: https://cloud.google.com/iam/docs/granting-roles-to-service-accounts#granting_access_to_a_service_account_for_a_resource diff --git a/cloud-build/index.js b/cloud-build/index.js deleted file mode 100644 index cdc2031cd646..000000000000 --- a/cloud-build/index.js +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2020 Google, LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const express = require('express'); -const app = express(); - -app.get('/', (req, res) => { - console.log('Hello world received a request.'); - - const target = process.env.TARGET || 'World'; - res.send(`Hello ${target}!`); -}); - -const port = process.env.PORT || 8080; -app.listen(port, () => { - console.log('Hello world listening on port', port); -}); diff --git a/cloud-build/package-lock.json b/cloud-build/package-lock.json deleted file mode 100644 index b8746679fd86..000000000000 --- a/cloud-build/package-lock.json +++ /dev/null @@ -1,374 +0,0 @@ -{ - "name": "helloworld", - "version": "1.0.0", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "accepts": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", - "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", - "requires": { - "mime-types": "~2.1.24", - "negotiator": "0.6.2" - } - }, - "array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" - }, - "body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", - "requires": { - "bytes": "3.1.0", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "1.7.2", - "iconv-lite": "0.4.24", - "on-finished": "~2.3.0", - "qs": "6.7.0", - "raw-body": "2.4.0", - "type-is": "~1.6.17" - } - }, - "bytes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", - "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" - }, - "content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", - "requires": { - "safe-buffer": "5.1.2" - } - }, - "content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" - }, - "cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" - }, - "cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" - }, - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" - }, - "destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" - }, - "ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" - }, - "encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" - }, - "escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" - }, - "etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" - }, - "express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", - "requires": { - "accepts": "~1.3.7", - "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", - "content-type": "~1.0.4", - "cookie": "0.4.0", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "~1.1.2", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.1.2", - "fresh": "0.5.2", - "merge-descriptors": "1.0.1", - "methods": "~1.1.2", - "on-finished": "~2.3.0", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", - "statuses": "~1.5.0", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - } - }, - "finalhandler": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", - "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", - "requires": { - "debug": "2.6.9", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "on-finished": "~2.3.0", - "parseurl": "~1.3.3", - "statuses": "~1.5.0", - "unpipe": "~1.0.0" - } - }, - "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" - }, - "fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" - }, - "http-errors": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", - "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", - "requires": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" - } - }, - "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" - }, - "media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" - }, - "merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" - }, - "methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" - }, - "mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" - }, - "mime-db": { - "version": "1.43.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", - "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==" - }, - "mime-types": { - "version": "2.1.26", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", - "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", - "requires": { - "mime-db": "1.43.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "negotiator": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", - "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" - }, - "on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", - "requires": { - "ee-first": "1.1.1" - } - }, - "parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" - }, - "path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" - }, - "proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", - "requires": { - "forwarded": "~0.1.2", - "ipaddr.js": "1.9.1" - } - }, - "qs": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", - "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" - }, - "range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" - }, - "raw-body": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", - "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", - "requires": { - "bytes": "3.1.0", - "http-errors": "1.7.2", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", - "requires": { - "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "~1.7.2", - "mime": "1.6.0", - "ms": "2.1.1", - "on-finished": "~2.3.0", - "range-parser": "~1.2.1", - "statuses": "~1.5.0" - }, - "dependencies": { - "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" - } - } - }, - "serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", - "requires": { - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.17.1" - } - }, - "setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" - }, - "statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" - }, - "toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" - }, - "type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", - "requires": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - } - }, - "unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" - }, - "utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" - }, - "vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" - } - } -} diff --git a/cloud-build/package.json b/cloud-build/package.json deleted file mode 100644 index b1f81256cc76..000000000000 --- a/cloud-build/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "helloworld", - "version": "1.0.0", - "description": "Simple hello world sample in Node", - "main": "index.js", - "scripts": { - "start": "node index.js" - }, - "author": "", - "license": "Apache-2.0", - "dependencies": { - "express": "^4.17.1" - } -} From 5247d3acaacdcdb06f94c4f383dbe5d5ceb3c73b Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Sun, 16 Oct 2022 13:57:28 -0700 Subject: [PATCH 177/269] Delete redundant terraform dir --- learning/tour-of-beam/terraform/README.md | 0 .../tour-of-beam/terraform/buckets/main.tf | 22 ------------- .../tour-of-beam/terraform/buckets/output.tf | 15 --------- .../terraform/buckets/variables.tf | 18 ---------- .../terraform/cloud-functions/main.tf | 23 ------------- .../terraform/cloud-functions/output.tf | 11 ------- .../terraform/cloud-functions/variables.tf | 7 ---- learning/tour-of-beam/terraform/main.tf | 24 -------------- learning/tour-of-beam/terraform/output.tf | 7 ---- learning/tour-of-beam/terraform/provider.tf | 19 ----------- learning/tour-of-beam/terraform/setup/iam.tf | 23 ------------- .../tour-of-beam/terraform/setup/output.tf | 3 -- .../tour-of-beam/terraform/setup/services.tf | 23 ------------- .../tour-of-beam/terraform/setup/variables.tf | 9 ----- learning/tour-of-beam/terraform/variables.tf | 33 ------------------- 15 files changed, 237 deletions(-) delete mode 100644 learning/tour-of-beam/terraform/README.md delete mode 100644 learning/tour-of-beam/terraform/buckets/main.tf delete mode 100644 learning/tour-of-beam/terraform/buckets/output.tf delete mode 100644 learning/tour-of-beam/terraform/buckets/variables.tf delete mode 100644 learning/tour-of-beam/terraform/cloud-functions/main.tf delete mode 100644 learning/tour-of-beam/terraform/cloud-functions/output.tf delete mode 100644 learning/tour-of-beam/terraform/cloud-functions/variables.tf delete mode 100644 learning/tour-of-beam/terraform/main.tf delete mode 100644 learning/tour-of-beam/terraform/output.tf delete mode 100644 learning/tour-of-beam/terraform/provider.tf delete mode 100644 learning/tour-of-beam/terraform/setup/iam.tf delete mode 100644 learning/tour-of-beam/terraform/setup/output.tf delete mode 100644 learning/tour-of-beam/terraform/setup/services.tf delete mode 100644 learning/tour-of-beam/terraform/setup/variables.tf delete mode 100644 learning/tour-of-beam/terraform/variables.tf diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/learning/tour-of-beam/terraform/buckets/main.tf b/learning/tour-of-beam/terraform/buckets/main.tf deleted file mode 100644 index c113197b5844..000000000000 --- a/learning/tour-of-beam/terraform/buckets/main.tf +++ /dev/null @@ -1,22 +0,0 @@ -resource "google_storage_bucket" "function_bucket" { - name = var.name - location = var.location - project = var.project_id - storage_class = var.storage_class -} - -resource "google_storage_bucket_object" "zip" { - # Use an MD5 here. If there's no changes to the source code, this won't change either. - # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing - # a redeployment when it has! - name = "${data.archive_file.source.output_md5}.zip" - bucket = google_storage_bucket.function_bucket.name - source = data.archive_file.source.output_path - content_type = "application/zip" -} - -resource "google_storage_bucket_access_control" "public_rule" { - bucket = google_storage_bucket.function_bucket.name - role = "READER" - entity = "allUsers" -} diff --git a/learning/tour-of-beam/terraform/buckets/output.tf b/learning/tour-of-beam/terraform/buckets/output.tf deleted file mode 100644 index 69bfc4f5c6fb..000000000000 --- a/learning/tour-of-beam/terraform/buckets/output.tf +++ /dev/null @@ -1,15 +0,0 @@ -output "function-bucket-id" { - value = google_storage_bucket.function_bucket.id -} - -output "function-bucket-name" { - value = google_storage_bucket.function_bucket.name -} - -output "function-bucket-object" { - value = google_storage_bucket_object.zip.name -} - -output "function-bucket-location" { - value = google_storage_bucket.function_bucket.location -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/buckets/variables.tf b/learning/tour-of-beam/terraform/buckets/variables.tf deleted file mode 100644 index 647c09f40620..000000000000 --- a/learning/tour-of-beam/terraform/buckets/variables.tf +++ /dev/null @@ -1,18 +0,0 @@ -variable "name" { - description = "Name of Bucket to Store Cloud Functions" - default = "tour-of-beam-cloudfunction-bucket" -} - -variable "location" { - description = "Cloud Functions Bucket Region" - default = "US" -} - -variable "project_id" { - description = "The GCP Project ID where ToB Cloud Functions will be created" -} - -variable "storage_class" { - description = "Functions Bucket Storage Class" - default = "STANDARD" -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/cloud-functions/main.tf b/learning/tour-of-beam/terraform/cloud-functions/main.tf deleted file mode 100644 index a926ef71c19e..000000000000 --- a/learning/tour-of-beam/terraform/cloud-functions/main.tf +++ /dev/null @@ -1,23 +0,0 @@ -module "setup" { - source = "../setup" -} - -module "buckets" { - source = "../buckets" -} - -resource "google_cloudfunctions_function" "cloud_function" { - name = "tour-of-beam-backend-cloud-function" - runtime = "go116" - region = var.region - service_account_email = module.setup.service-account-email - ingress_settings = "ALLOW_ALL" - # Get the source code of the cloud function as a Zip compression - source_archive_bucket = module.buckets.function-bucket-name - source_archive_object = module.buckets.function-bucket-object - - trigger_http = true - # Name of the function that will be executed when the Google Cloud Function is triggered (def hello_gcs) - entry_point = "init" - -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/cloud-functions/output.tf b/learning/tour-of-beam/terraform/cloud-functions/output.tf deleted file mode 100644 index 1d42a873e283..000000000000 --- a/learning/tour-of-beam/terraform/cloud-functions/output.tf +++ /dev/null @@ -1,11 +0,0 @@ -output "cloud-function-trigger-url" { - value = google_cloudfunctions_function.cloud_function.https_trigger_url -} - -output "cloud-function-name" { - value = google_cloudfunctions_function.cloud_function.name -} - -output "cloud-function-region" { - value = google_cloudfunctions_function.cloud_function.region -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/cloud-functions/variables.tf b/learning/tour-of-beam/terraform/cloud-functions/variables.tf deleted file mode 100644 index 136535b06765..000000000000 --- a/learning/tour-of-beam/terraform/cloud-functions/variables.tf +++ /dev/null @@ -1,7 +0,0 @@ -variable "project_id" { - description = "The GCP Project ID where ToB Cloud Functions will be created" -} - -variable "region" { - description = "Region of App" -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf deleted file mode 100644 index 1fda102f7e89..000000000000 --- a/learning/tour-of-beam/terraform/main.tf +++ /dev/null @@ -1,24 +0,0 @@ -module "setup" { - depends_on = [module.setup] - source = "./setup" - project_id = var.project_id - service_account_id = var.service_account_id -} - - -module "buckets" { - depends_on = [module.setup] - source = "./buckets" - project_id = var.project_id - name = var.function_bucket_name - storage_class = var.function_bucket_storage_class - location = var.function_bucket_location -} - -module "cloud-functions" { - depends_on = [module.buckets, module.setup] - source = "./cloud-functions" - project_id = var.project_id - service_account_email = module.setup.service-account-email - region = var.region -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/output.tf b/learning/tour-of-beam/terraform/output.tf deleted file mode 100644 index d77e377b6823..000000000000 --- a/learning/tour-of-beam/terraform/output.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "function-bucket-id" { - value = module.buckets.function-bucket-id -} - -output "cloud-function-trigger-url" { - value = module.cloud-functions.cloud-function-trigger-url -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/provider.tf b/learning/tour-of-beam/terraform/provider.tf deleted file mode 100644 index e91570b1793a..000000000000 --- a/learning/tour-of-beam/terraform/provider.tf +++ /dev/null @@ -1,19 +0,0 @@ -terraform { - # this describe buket for save state playground cloud - backend "gcs" { - } - - required_providers { - google = { - source = "hashicorp/google" - version = "4.0.0" - } - } -} - -provider "google" { - region = var.region - project = var.project_id - // TODO may need to run module.setup first independent of this solution and add the terraform service account as a variable - // This allows us to use a service account to provision resources without downloading or storing service account keys -} diff --git a/learning/tour-of-beam/terraform/setup/iam.tf b/learning/tour-of-beam/terraform/setup/iam.tf deleted file mode 100644 index 835add0e2468..000000000000 --- a/learning/tour-of-beam/terraform/setup/iam.tf +++ /dev/null @@ -1,23 +0,0 @@ -resource "google_service_account" "sa_cloud_function" { - account_id = var.service_account_id - display_name = "Service Account to run Cloud Functions" - project = var.project_id -} - -resource "google_project_iam_member" "terraform_service_account_roles" { - for_each = toset([ - "roles/cloudfunctions.developer", "roles/storage.objectViewer", "roles/storage.objectCreator", - ]) - role = each.key - member = "serviceAccount:${google_service_account.sa_cloud_function.email}" - project = var.project_id -} - -#resource "google_cloudfunctions_function_iam_member" "invoker" { -# project = var.project_id -# region = module.cloud-functions.cloud-function-region -# cloud_function = module.cloud-functions.cloud-function-name -# -# role = "roles/cloudfunctions.invoker" -# member = "allUsers" -#} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/output.tf b/learning/tour-of-beam/terraform/setup/output.tf deleted file mode 100644 index cf9c64097f91..000000000000 --- a/learning/tour-of-beam/terraform/setup/output.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "service-account-email" { - value = google_service_account.sa_cloud_function.email -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/services.tf b/learning/tour-of-beam/terraform/setup/services.tf deleted file mode 100644 index 887baef74cc9..000000000000 --- a/learning/tour-of-beam/terraform/setup/services.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Enable API for Cloud Build -resource "google_project_service" "cloud_build" { - project = var.project_id - service = "cloudbuild.googleapis.com" -} - -# Enable API for Cloud Function -resource "google_project_service" "cloud_function" { - project = var.project_id - service = "cloudfunctions.googleapis.com" -} - -# Enable API for Resource Manager -resource "google_project_service" "resource_manager" { - project = var.project_id - service = "cloudresourcemanager.googleapis.com" -} - -# Enable API for IAM -resource "google_project_service" "iam" { - project = var.project_id - service = "iam.googleapis.com" -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/variables.tf b/learning/tour-of-beam/terraform/setup/variables.tf deleted file mode 100644 index f5ccf641c91f..000000000000 --- a/learning/tour-of-beam/terraform/setup/variables.tf +++ /dev/null @@ -1,9 +0,0 @@ -variable "project_id" { - description = "The GCP Project ID where ToB Cloud Functions will be created" -} - -variable "service_account_id" { - description = "Service account ID" - default = "tour-of-beam-sa" -} - diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf deleted file mode 100644 index 9bcedf386afe..000000000000 --- a/learning/tour-of-beam/terraform/variables.tf +++ /dev/null @@ -1,33 +0,0 @@ -#COMMON - -variable "project_id" { - description = "The GCP Project ID where ToB Cloud Functions will be created" -} - -variable "region" { - description = "Infrastructure Region" -} - -#IAM - -variable "service_account_id" { - description = "Service account ID" - default = "tour-of-beam-sa" -} - -#Buckets - -variable "function_bucket_name" { - description = "Name of Bucket to Store Cloud Functions" - default = "tour-of-beam-cloudfunction-bucket" -} - -variable "function_bucket_location" { - description = "Cloud Functions Bucket Region" - default = "US" -} - -variable "function_bucket_storage_class" { - description = "Functions Bucket Storage Class" - default = "STANDARD" -} \ No newline at end of file From a331cb5be9fa1b1aeefcec5615668e913140b31e Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Sun, 16 Oct 2022 13:58:16 -0700 Subject: [PATCH 178/269] Delete state.tfbackend --- .../environment/beta/state.tfbackend | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 playground/terraform/environment/beta/state.tfbackend diff --git a/playground/terraform/environment/beta/state.tfbackend b/playground/terraform/environment/beta/state.tfbackend deleted file mode 100644 index b2c574de8e4b..000000000000 --- a/playground/terraform/environment/beta/state.tfbackend +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -bucket = "pg-second" From 33aa2b7c81a3d3062fe5161d870d4d5e7ded098a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Sun, 16 Oct 2022 14:25:56 -0700 Subject: [PATCH 179/269] Update main.tf --- .../tour-of-beam/terraform-v2/modules/cloud_functions/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 866e1f5a33fa..74adb3e24ff5 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -1,6 +1,6 @@ resource "google_cloudfunctions_function" "cloud_function" { count = length(var.entry_point_names) - name = "tour-of-beam-backend-cloud-function-${count.index}" + name = "${var.entry_point_names[count.index]}" runtime = "go116" project = var.project_id service_account_email = var.service_account_id From 78400c25ebbbcb5d91ba4a094b5006845789128f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Oct 2022 08:27:46 -0700 Subject: [PATCH 180/269] Move files --- .../terraform-v2/{environments/uat => }/backend.tf | 0 learning/tour-of-beam/terraform-v2/cloudbuild.yaml | 12 ++++++++++-- .../terraform-v2/{environments/uat => }/main.tf | 12 ++++-------- .../terraform-v2/{environments/uat => }/outputs.tf | 0 .../terraform-v2/{environments/uat => }/variables.tf | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) rename learning/tour-of-beam/terraform-v2/{environments/uat => }/backend.tf (100%) rename learning/tour-of-beam/terraform-v2/{environments/uat => }/main.tf (80%) rename learning/tour-of-beam/terraform-v2/{environments/uat => }/outputs.tf (100%) rename learning/tour-of-beam/terraform-v2/{environments/uat => }/variables.tf (90%) diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/backend.tf b/learning/tour-of-beam/terraform-v2/backend.tf similarity index 100% rename from learning/tour-of-beam/terraform-v2/environments/uat/backend.tf rename to learning/tour-of-beam/terraform-v2/backend.tf diff --git a/learning/tour-of-beam/terraform-v2/cloudbuild.yaml b/learning/tour-of-beam/terraform-v2/cloudbuild.yaml index 57f36ee00bca..6d173ea114bf 100644 --- a/learning/tour-of-beam/terraform-v2/cloudbuild.yaml +++ b/learning/tour-of-beam/terraform-v2/cloudbuild.yaml @@ -22,6 +22,8 @@ steps: echo "***********************" echo "$BRANCH_NAME" echo "***********************" + options: + logging: 'CLOUD_LOGGING_ONLY' - id: 'tf init' name: 'hashicorp/terraform:1.3.1' @@ -45,7 +47,9 @@ steps: terraform init || exit 1 cd ../../ done - fi + fi + options: + logging: 'CLOUD_LOGGING_ONLY' # [START tf-plan] - id: 'tf plan' @@ -71,6 +75,8 @@ steps: cd ../../ done fi + options: + logging: 'CLOUD_LOGGING_ONLY' # [END tf-plan] # [START tf-apply] @@ -88,4 +94,6 @@ steps: echo "Branch '$BRANCH_NAME' does not represent an official environment." echo "*******************************************************************************" fi -# [END tf-apply] \ No newline at end of file + options: + logging: 'CLOUD_LOGGING_ONLY' +# [END tf-apply] diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf b/learning/tour-of-beam/terraform-v2/main.tf similarity index 80% rename from learning/tour-of-beam/terraform-v2/environments/uat/main.tf rename to learning/tour-of-beam/terraform-v2/main.tf index f8ee79915e43..ac9740fc49c4 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/main.tf +++ b/learning/tour-of-beam/terraform-v2/main.tf @@ -1,32 +1,28 @@ -locals { - env = "uat" -} - provider "google" { project = var.project_id } module "iam" { - source = "../../modules/iam" + source = "modules/iam" project_id = var.project_id service_account_id = var.service_account_id depends_on = [module.api_enable] } module "buckets" { - source = "../../modules/buckets" + source = "modules/buckets" project_id = var.project_id bucket_name = var.bucket_name depends_on = [module.iam, module.api_enable] } module "api_enable" { - source = "../../modules/api_enable" + source = "modules/api_enable" project_id = var.project_id } module "cloud_functions" { - source = "../../modules/cloud_functions" + source = "modules/cloud_functions" region = var.region project_id = var.project_id bucket_name = var.bucket_name diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf b/learning/tour-of-beam/terraform-v2/outputs.tf similarity index 100% rename from learning/tour-of-beam/terraform-v2/environments/uat/outputs.tf rename to learning/tour-of-beam/terraform-v2/outputs.tf diff --git a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf b/learning/tour-of-beam/terraform-v2/variables.tf similarity index 90% rename from learning/tour-of-beam/terraform-v2/environments/uat/variables.tf rename to learning/tour-of-beam/terraform-v2/variables.tf index 22dd45569e2c..6812772f2285 100644 --- a/learning/tour-of-beam/terraform-v2/environments/uat/variables.tf +++ b/learning/tour-of-beam/terraform-v2/variables.tf @@ -8,5 +8,5 @@ variable "service_account_id" { description = "Name of SA to run Cloud Function" } variable "region" { - default = "europe-west1" + default = "us-central1" } \ No newline at end of file From 7816bac8fa7266d641ced968a4b80831ff75f425 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Oct 2022 09:04:09 -0700 Subject: [PATCH 181/269] Update cloudbuild.yaml --- learning/tour-of-beam/terraform-v2/cloudbuild.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/cloudbuild.yaml b/learning/tour-of-beam/terraform-v2/cloudbuild.yaml index 6d173ea114bf..3bc602ffbc5f 100644 --- a/learning/tour-of-beam/terraform-v2/cloudbuild.yaml +++ b/learning/tour-of-beam/terraform-v2/cloudbuild.yaml @@ -22,8 +22,6 @@ steps: echo "***********************" echo "$BRANCH_NAME" echo "***********************" - options: - logging: 'CLOUD_LOGGING_ONLY' - id: 'tf init' name: 'hashicorp/terraform:1.3.1' @@ -48,8 +46,6 @@ steps: cd ../../ done fi - options: - logging: 'CLOUD_LOGGING_ONLY' # [START tf-plan] - id: 'tf plan' @@ -75,8 +71,6 @@ steps: cd ../../ done fi - options: - logging: 'CLOUD_LOGGING_ONLY' # [END tf-plan] # [START tf-apply] @@ -94,6 +88,6 @@ steps: echo "Branch '$BRANCH_NAME' does not represent an official environment." echo "*******************************************************************************" fi - options: - logging: 'CLOUD_LOGGING_ONLY' # [END tf-apply] +options: + logging: CLOUD_LOGGING_ONLY \ No newline at end of file From 19af091cab7c15ded185b47669deeaafec383db4 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:05:19 -0700 Subject: [PATCH 182/269] Create gcp-deploy.yml --- .github/workflows/gcp-deploy.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/gcp-deploy.yml diff --git a/.github/workflows/gcp-deploy.yml b/.github/workflows/gcp-deploy.yml new file mode 100644 index 000000000000..5eccb46b8f1e --- /dev/null +++ b/.github/workflows/gcp-deploy.yml @@ -0,0 +1,24 @@ +name: GCP Cloud Build + +job: + job_id: + name: Trigger Google Cloud Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - id: 'auth' + uses: 'google-github-actions/auth@v0' + with: + credentials_json: '${{ secrets.GCP_CREDENTIALS }}' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v0' + + - name: 'Use gcloud CLI' + run: 'gcloud info' + + - name: 'Start Google Cloud Build trigger' + run: | + export CLOUDSDK_CORE_DISABLE_PROMPTS=1 + gcloud beta builds triggers run ${{ secrets.GCP_BUILD_STAGING_TRIGGER_ID }} --tag=${{ github.event.release.tag_name }} From b2c91e6be7a4b66b8d2d86b35e7f213c86f89875 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:09:24 -0700 Subject: [PATCH 183/269] Update gcp-deploy.yml --- .github/workflows/gcp-deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gcp-deploy.yml b/.github/workflows/gcp-deploy.yml index 5eccb46b8f1e..b3d4d0d39754 100644 --- a/.github/workflows/gcp-deploy.yml +++ b/.github/workflows/gcp-deploy.yml @@ -1,6 +1,8 @@ name: GCP Cloud Build -job: +on: workflow_dispatch + +jobs: job_id: name: Trigger Google Cloud Build runs-on: ubuntu-latest From 10b5a3eba1b904f2f33ece616839dd8d831a6531 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Oct 2022 23:11:48 -0700 Subject: [PATCH 184/269] Update main.tf --- learning/tour-of-beam/terraform-v2/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/learning/tour-of-beam/terraform-v2/main.tf b/learning/tour-of-beam/terraform-v2/main.tf index ac9740fc49c4..f49aba6a964f 100644 --- a/learning/tour-of-beam/terraform-v2/main.tf +++ b/learning/tour-of-beam/terraform-v2/main.tf @@ -1,3 +1,7 @@ +locals { + env = "uat" +} + provider "google" { project = var.project_id } From 9de2cd3073f8f24dad4d828de76850bafc63fd40 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 20 Oct 2022 07:57:28 -0700 Subject: [PATCH 185/269] Update TF script for TOB backend --- learning/tour-of-beam/terraform-v2/backend.tf | 6 -- .../terraform-v2/backend_state.tf | 12 +++ .../tour-of-beam/terraform-v2/cloudbuild.yaml | 93 ------------------- .../modules/api_enable/variables.tf | 4 +- .../terraform-v2/modules/buckets/main.tf | 10 +- .../terraform-v2/modules/buckets/outputs.tf | 4 +- .../terraform-v2/modules/buckets/variables.tf | 8 +- .../terraform-v2/modules/iam/main.tf | 6 +- .../terraform-v2/modules/iam/outputs.tf | 2 +- .../terraform-v2/modules/iam/variables.tf | 4 +- .../tour-of-beam/terraform-v2/variables.tf | 15 ++- 11 files changed, 41 insertions(+), 123 deletions(-) delete mode 100644 learning/tour-of-beam/terraform-v2/backend.tf create mode 100644 learning/tour-of-beam/terraform-v2/backend_state.tf delete mode 100644 learning/tour-of-beam/terraform-v2/cloudbuild.yaml diff --git a/learning/tour-of-beam/terraform-v2/backend.tf b/learning/tour-of-beam/terraform-v2/backend.tf deleted file mode 100644 index 4131f13934e5..000000000000 --- a/learning/tour-of-beam/terraform-v2/backend.tf +++ /dev/null @@ -1,6 +0,0 @@ -terraform { - backend "gcs" { - bucket = "tour-of-beam-backend-tfstate-bucket" - prefix = "env/uat" - } -} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/backend_state.tf b/learning/tour-of-beam/terraform-v2/backend_state.tf new file mode 100644 index 000000000000..85723beb13a8 --- /dev/null +++ b/learning/tour-of-beam/terraform-v2/backend_state.tf @@ -0,0 +1,12 @@ +terraform { + backend "gcs" { + bucket = "tour-of-beam-backend-tfstate-bucket" + prefix = "terraform-state" + } + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.40.0" + } + } +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/cloudbuild.yaml b/learning/tour-of-beam/terraform-v2/cloudbuild.yaml deleted file mode 100644 index 3bc602ffbc5f..000000000000 --- a/learning/tour-of-beam/terraform-v2/cloudbuild.yaml +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -steps: - - id: 'branch name' - name: 'alpine' - entrypoint: 'sh' - args: - - '-c' - - | - echo "***********************" - echo "$BRANCH_NAME" - echo "***********************" - - - id: 'tf init' - name: 'hashicorp/terraform:1.3.1' - entrypoint: 'sh' - args: - - '-c' - - | - if [ -d "environments/$BRANCH_NAME/" ]; then - cd environments/$BRANCH_NAME - terraform init - else - for dir in environments/*/ - do - cd ${dir} - env=${dir%*/} - env=${env#*/} - echo "" - echo "*************** TERRAFORM INIT ******************" - echo "******* At environment: ${env} ********" - echo "*************************************************" - terraform init || exit 1 - cd ../../ - done - fi - - # [START tf-plan] - - id: 'tf plan' - name: 'hashicorp/terraform:1.3.1' - entrypoint: 'sh' - args: - - '-c' - - | - if [ -d "environments/$BRANCH_NAME/" ]; then - cd environments/$BRANCH_NAME - terraform plan - else - for dir in environments/*/ - do - cd ${dir} - env=${dir%*/} - env=${env#*/} - echo "" - echo "*************** TERRAFOM PLAN ******************" - echo "******* At environment: ${env} ********" - echo "*************************************************" - terraform plan || exit 1 - cd ../../ - done - fi - # [END tf-plan] - - # [START tf-apply] - - id: 'tf apply' - name: 'hashicorp/terraform:1.3.1' - entrypoint: 'sh' - args: - - '-c' - - | - if [ -d "environments/$BRANCH_NAME/" ]; then - cd environments/$BRANCH_NAME - terraform apply -auto-approve - else - echo "***************************** SKIPPING APPLYING *******************************" - echo "Branch '$BRANCH_NAME' does not represent an official environment." - echo "*******************************************************************************" - fi -# [END tf-apply] -options: - logging: CLOUD_LOGGING_ONLY \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf index 6911556306ea..7488e265382a 100644 --- a/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf @@ -1 +1,3 @@ -variable "project_id" {} \ No newline at end of file +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf index 5ab4734a75c3..43ae354c67fc 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -1,4 +1,4 @@ -resource "google_storage_bucket" "functions_bucket" { +resource "google_storage_bucket" "cloud_functions_bucket" { name = var.bucket_name location = var.location project = var.project_id @@ -10,13 +10,7 @@ resource "google_storage_bucket_object" "zip" { # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing # a redeployment when it has! name = "${data.archive_file.source.output_md5}.zip" - bucket = google_storage_bucket.functions_bucket.name + bucket = google_storage_bucket.cloud_functions_bucket.name source = data.archive_file.source.output_path content_type = "application/zip" -} - -resource "google_storage_bucket_access_control" "public_rule" { - bucket = google_storage_bucket.functions_bucket.name - role = "READER" - entity = "allUsers" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf index bffafd3c2d56..4841dc829edb 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf @@ -1,9 +1,9 @@ output "functions-bucket-id" { - value = google_storage_bucket.functions_bucket.id + value = google_storage_bucket.cloud_functions_bucket.id } output "functions-bucket-name" { - value = google_storage_bucket.functions_bucket.name + value = google_storage_bucket.cloud_functions_bucket.name } output "function-bucket-object" { diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf index 0416cc20cbfd..38d3f58656d2 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -1,15 +1,15 @@ #Generates archive of source code variable "bucket_name" { - description = "Bucket name to store function source code" + description = "The bucket name to store functions' source code" } variable "location" { - description = "Cloud Functions Bucket Region" - default = "europe-west1" + description = "Cloud Functions bucket Region" + default = "us-central1" } variable "project_id" { - description = "Our GCP Project" + description = "The ID of the Google Cloud project within which resources are provisioned" } data "archive_file" "source" { diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index e5a0e62027aa..148a78c4398e 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -1,15 +1,15 @@ -resource "google_service_account" "tourofbeam_cf_sa" { +resource "google_service_account" "cloud_function_sa" { account_id = var.service_account_id display_name = "Service Account to run Cloud Functions" } resource "google_project_iam_member" "terraform_service_account_roles" { for_each = toset([ - "roles/resourcemanager.projectIamAdmin", "roles/cloudfunctions.admin", "roles/storage.objectViewer", + "roles/cloudfunctions.admin", "roles/storage.objectViewer", "roles/storage.objectCreator", "roles/iam.serviceAccountUser" ]) role = each.key - member = "serviceAccount:${google_service_account.tourofbeam_cf_sa.email}" + member = "serviceAccount:${google_service_account.cloud_function_sa.email}" project = var.project_id } diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf index 0a5da6d2e0d4..57285172fce6 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf @@ -1,3 +1,3 @@ output "service-account-email" { - value = google_service_account.tourofbeam_cf_sa.email + value = google_service_account.cloud_function_sa.email } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf index 8ef947b0f3ed..ac40f8352b60 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -1,8 +1,8 @@ variable "project_id" { - description = "Our GCP Project" + description = "The ID of the Google Cloud project within which resources are provisioned" } variable "service_account_id" { - description = "Name of SA to run Cloud Function" + description = "The ID of the service account responsible for running Google Cloud functions" } diff --git a/learning/tour-of-beam/terraform-v2/variables.tf b/learning/tour-of-beam/terraform-v2/variables.tf index 6812772f2285..653d1c237d39 100644 --- a/learning/tour-of-beam/terraform-v2/variables.tf +++ b/learning/tour-of-beam/terraform-v2/variables.tf @@ -1,11 +1,20 @@ variable "bucket_name" { - description = "Bucket name to store function source code" + description = "The bucket name that will store functions' source code" } variable "project_id" { - description = "Our GCP Project" + type = string + description = "The ID of the Google Cloud project within which resources are provisioned" } + +variable "iac_service_account_id" { + type = string + description = "The ID of the service account responsible for provisioning Google Cloud resources using Infrastructure-as-Code" + default = "terraform" +} + variable "service_account_id" { - description = "Name of SA to run Cloud Function" + type = string + description = "The ID of the service account responsible for running Google Cloud functions" } variable "region" { default = "us-central1" From b90bfdd2203b6258b5f32f2b5480bd34b06a5c72 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 21 Oct 2022 01:18:18 -0700 Subject: [PATCH 186/269] Delete gcp-deploy.yml --- .github/workflows/gcp-deploy.yml | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 .github/workflows/gcp-deploy.yml diff --git a/.github/workflows/gcp-deploy.yml b/.github/workflows/gcp-deploy.yml deleted file mode 100644 index b3d4d0d39754..000000000000 --- a/.github/workflows/gcp-deploy.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: GCP Cloud Build - -on: workflow_dispatch - -jobs: - job_id: - name: Trigger Google Cloud Build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - id: 'auth' - uses: 'google-github-actions/auth@v0' - with: - credentials_json: '${{ secrets.GCP_CREDENTIALS }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v0' - - - name: 'Use gcloud CLI' - run: 'gcloud info' - - - name: 'Start Google Cloud Build trigger' - run: | - export CLOUDSDK_CORE_DISABLE_PROMPTS=1 - gcloud beta builds triggers run ${{ secrets.GCP_BUILD_STAGING_TRIGGER_ID }} --tag=${{ github.event.release.tag_name }} From c8cdc0a49666e76204f7fedee453da7e8744a5c9 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 21 Oct 2022 01:22:11 -0700 Subject: [PATCH 187/269] Update main.tf --- .../tour-of-beam/terraform-v2/modules/cloud_functions/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 74adb3e24ff5..45ceb8c5f3b3 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -2,6 +2,7 @@ resource "google_cloudfunctions_function" "cloud_function" { count = length(var.entry_point_names) name = "${var.entry_point_names[count.index]}" runtime = "go116" + available_memory_mb = 128 project = var.project_id service_account_email = var.service_account_id source_archive_bucket = var.source_archive_bucket From 22f9d65c831245c53b14bb13deef2b0ef15fcde1 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 21 Oct 2022 03:08:25 -0700 Subject: [PATCH 188/269] Final updates. Ready for review --- learning/tour-of-beam/terraform-v2/main.tf | 8 ++++---- .../tour-of-beam/terraform-v2/modules/api_enable/main.tf | 4 ++++ .../tour-of-beam/terraform-v2/modules/buckets/main.tf | 4 ++-- .../terraform-v2/modules/buckets/variables.tf | 4 ++-- learning/tour-of-beam/terraform-v2/variables.tf | 2 ++ 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/main.tf b/learning/tour-of-beam/terraform-v2/main.tf index f49aba6a964f..d8b51df00b35 100644 --- a/learning/tour-of-beam/terraform-v2/main.tf +++ b/learning/tour-of-beam/terraform-v2/main.tf @@ -7,26 +7,26 @@ provider "google" { } module "iam" { - source = "modules/iam" + source = "./modules/iam" project_id = var.project_id service_account_id = var.service_account_id depends_on = [module.api_enable] } module "buckets" { - source = "modules/buckets" + source = "./modules/buckets" project_id = var.project_id bucket_name = var.bucket_name depends_on = [module.iam, module.api_enable] } module "api_enable" { - source = "modules/api_enable" + source = "./modules/api_enable" project_id = var.project_id } module "cloud_functions" { - source = "modules/cloud_functions" + source = "./modules/cloud_functions" region = var.region project_id = var.project_id bucket_name = var.bucket_name diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf index 887baef74cc9..a0fb23a12a95 100644 --- a/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf @@ -2,22 +2,26 @@ resource "google_project_service" "cloud_build" { project = var.project_id service = "cloudbuild.googleapis.com" + disable_on_destroy = false } # Enable API for Cloud Function resource "google_project_service" "cloud_function" { project = var.project_id service = "cloudfunctions.googleapis.com" + disable_on_destroy = false } # Enable API for Resource Manager resource "google_project_service" "resource_manager" { project = var.project_id service = "cloudresourcemanager.googleapis.com" + disable_on_destroy = false } # Enable API for IAM resource "google_project_service" "iam" { project = var.project_id service = "iam.googleapis.com" + disable_on_destroy = false } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf index 43ae354c67fc..85f100e4672d 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -9,8 +9,8 @@ resource "google_storage_bucket_object" "zip" { # Use an MD5 here. If there's no changes to the source code, this won't change either. # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing # a redeployment when it has! - name = "${data.archive_file.source.output_md5}.zip" + name = "${data.archive_file.source_code.output_md5}.zip" bucket = google_storage_bucket.cloud_functions_bucket.name - source = data.archive_file.source.output_path + source = data.archive_file.source_code.output_path content_type = "application/zip" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf index 38d3f58656d2..e3522f2c0c39 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -12,8 +12,8 @@ variable "project_id" { description = "The ID of the Google Cloud project within which resources are provisioned" } -data "archive_file" "source" { +data "archive_file" "source_code" { type = "zip" - source_dir = "../../../backend" + source_dir = "../backend" output_path = "/tmp/backend.zip" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/variables.tf b/learning/tour-of-beam/terraform-v2/variables.tf index 653d1c237d39..55e10baf164e 100644 --- a/learning/tour-of-beam/terraform-v2/variables.tf +++ b/learning/tour-of-beam/terraform-v2/variables.tf @@ -1,6 +1,7 @@ variable "bucket_name" { description = "The bucket name that will store functions' source code" } + variable "project_id" { type = string description = "The ID of the Google Cloud project within which resources are provisioned" @@ -16,6 +17,7 @@ variable "service_account_id" { type = string description = "The ID of the service account responsible for running Google Cloud functions" } + variable "region" { default = "us-central1" } \ No newline at end of file From 1280d4d99e5f935a56703577d0f235fa84c6cd92 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 21 Oct 2022 03:48:33 -0700 Subject: [PATCH 189/269] Deleted locals variable --- learning/tour-of-beam/terraform-v2/main.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/main.tf b/learning/tour-of-beam/terraform-v2/main.tf index d8b51df00b35..da89eb863e1d 100644 --- a/learning/tour-of-beam/terraform-v2/main.tf +++ b/learning/tour-of-beam/terraform-v2/main.tf @@ -1,7 +1,3 @@ -locals { - env = "uat" -} - provider "google" { project = var.project_id } From 62c8379c3c6bf70b62b9b923b31fa47ca5ce212c Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 09:38:58 -0700 Subject: [PATCH 190/269] Cloud Build for PG Backend (go, java) --- .../cloudbuild_playground_backend.yml | 40 ++++++++++++++++++ .../dockerfiles/Dockerfile-base-image-backend | 41 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 gha-cloudbuild/cloudbuild_playground_backend.yml create mode 100644 gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml new file mode 100644 index 000000000000..62659f7b1d3b --- /dev/null +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + # This step builds the base image for Beam Playground backend builds + - name: 'gcr.io/cloud-builders/docker' + args: + - 'build' + - '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID-builder:latest' + - '-f' + - 'dockerfiles/Dockerfile-base-image-backend' + - '.' + images: ['$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID-builder:latest'] + + # This step builds Beam Playground backend container for golang + - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID-builder:latest' + entrypoint: './gradlew' + args: + - ':playground:backend:containers:go:docker' + + # This step builds Beam Playground backend container for java + - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID-builder:latest' + entrypoint: './gradlew' + args: + - ':playground:backend:containers:java:docker' + +substitutions: + _ARTIFACT_REGISTRY_REPOSITORY_ID: playground_repository diff --git a/gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend b/gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend new file mode 100644 index 000000000000..a0f935287a64 --- /dev/null +++ b/gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend @@ -0,0 +1,41 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +FROM gcr.io/cloud-builders/gcloud + +ARG GO_VERSION=1.19.2 +ENV PATH="${PATH}:/usr/local/go/bin" + +RUN apt-get update >/dev/null + +# Install Docker +# See: https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script +RUN curl -fsSL https://get.docker.com -o get-docker.sh +RUN sh ./get-docker.sh >/dev/null + +# Install Go +# See: https://go.dev/doc/install +RUN curl -OL https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz +RUN sha256sum go$GO_VERSION.linux-amd64.tar.gz +RUN tar -C /usr/local -xvf go$GO_VERSION.linux-amd64.tar.gz +RUN rm go$GO_VERSION.linux-amd64.tar.gz +RUN go version + +# Install Java +RUN apt-get install openjdk-11-jdk -y >/dev/null +RUN java -version \ No newline at end of file From 9b18af7b3dda87db650b4ee203b8cc83ffdd85c4 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 09:47:42 -0700 Subject: [PATCH 191/269] Update cloudbuild_playground_backend.yml --- gha-cloudbuild/cloudbuild_playground_backend.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index 62659f7b1d3b..86d902302dfe 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -18,20 +18,22 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: - 'build' - - '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID-builder:latest' + - '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - '-f' - 'dockerfiles/Dockerfile-base-image-backend' - '.' - images: ['$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID-builder:latest'] + +images: + - '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' # This step builds Beam Playground backend container for golang - - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID-builder:latest' + - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' entrypoint: './gradlew' args: - ':playground:backend:containers:go:docker' # This step builds Beam Playground backend container for java - - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID-builder:latest' + - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' entrypoint: './gradlew' args: - ':playground:backend:containers:java:docker' From 438c3298f673afadad1091f0aafb8ea71de7672e Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 09:49:17 -0700 Subject: [PATCH 192/269] Update cloudbuild_playground_backend.yml --- gha-cloudbuild/cloudbuild_playground_backend.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index 86d902302dfe..37d3ccec328c 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -23,9 +23,6 @@ steps: - 'dockerfiles/Dockerfile-base-image-backend' - '.' -images: - - '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - # This step builds Beam Playground backend container for golang - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' entrypoint: './gradlew' @@ -40,3 +37,6 @@ images: substitutions: _ARTIFACT_REGISTRY_REPOSITORY_ID: playground_repository + +images: + - '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' \ No newline at end of file From ade8c63e7c00c4fe02a942e8d294c9b15673051d Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 09:52:11 -0700 Subject: [PATCH 193/269] Update cloudbuild_playground_backend.yml --- gha-cloudbuild/cloudbuild_playground_backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index 37d3ccec328c..e9a88b7f0a3c 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -20,7 +20,7 @@ steps: - 'build' - '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - '-f' - - 'dockerfiles/Dockerfile-base-image-backend' + - 'gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend' - '.' # This step builds Beam Playground backend container for golang From c3b0b0405dea8357a310d47cdf122f2e427a9ddd Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 10:09:35 -0700 Subject: [PATCH 194/269] Update cloudbuild_playground_backend.yml --- gha-cloudbuild/cloudbuild_playground_backend.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index e9a88b7f0a3c..144a931c041d 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -18,19 +18,19 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: - 'build' - - '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' + - '--tag=us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - '-f' - 'gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend' - '.' # This step builds Beam Playground backend container for golang - - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' + - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' entrypoint: './gradlew' args: - ':playground:backend:containers:go:docker' # This step builds Beam Playground backend container for java - - name: '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' + - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' entrypoint: './gradlew' args: - ':playground:backend:containers:java:docker' @@ -39,4 +39,4 @@ substitutions: _ARTIFACT_REGISTRY_REPOSITORY_ID: playground_repository images: - - '$LOCATION-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' \ No newline at end of file + - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' \ No newline at end of file From cf7c42c1823057f41c4e2b4afd6f9c900c3eb3ed Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 10:28:39 -0700 Subject: [PATCH 195/269] Update cloudbuild_playground_backend.yml --- gha-cloudbuild/cloudbuild_playground_backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index 144a931c041d..5b4e7ae93324 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -36,7 +36,7 @@ steps: - ':playground:backend:containers:java:docker' substitutions: - _ARTIFACT_REGISTRY_REPOSITORY_ID: playground_repository + _ARTIFACT_REGISTRY_REPOSITORY_ID: playground-repository images: - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' \ No newline at end of file From 348a0a5eba8d34df70d3ad9db2090c9665cec3ad Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 11:31:50 -0700 Subject: [PATCH 196/269] Update cloudbuild_playground_backend.yml --- gha-cloudbuild/cloudbuild_playground_backend.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index 5b4e7ae93324..7bd361fec022 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -39,4 +39,8 @@ substitutions: _ARTIFACT_REGISTRY_REPOSITORY_ID: playground-repository images: - - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' \ No newline at end of file + - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' + - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/go:latest' + - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/java:latest' + +timeout: 1800s \ No newline at end of file From 68bd4a0bc5a1d66ccd4625433d115b89ca6b8186 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 11:32:04 -0700 Subject: [PATCH 197/269] Update cloudbuild_playground_backend.yml --- gha-cloudbuild/cloudbuild_playground_backend.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index 7bd361fec022..1d026777107d 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -40,7 +40,5 @@ substitutions: images: - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/go:latest' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/java:latest' timeout: 1800s \ No newline at end of file From 4cf12b5ea95ff8d46456b960940a211789652e03 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 12:10:57 -0700 Subject: [PATCH 198/269] Update cloudbuild_playground_backend.yml --- .../cloudbuild_playground_backend.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index 1d026777107d..ef65ab2da114 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -35,6 +35,26 @@ steps: args: - ':playground:backend:containers:java:docker' + # This step builds Beam Playground backend container for java + - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' + entrypoint: './gradlew' + args: + - ':playground:backend:containers:python:docker' + + + # This step builds Beam Playground backend container for java + - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' + entrypoint: './gradlew' + args: + - ':playground:backend:containers:router:docker' + + + # This step builds Beam Playground backend container for java + - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' + entrypoint: './gradlew' + args: + - ':playground:backend:containers:scio:docker' + substitutions: _ARTIFACT_REGISTRY_REPOSITORY_ID: playground-repository From 360ab7ff9433937dce8475d6f60101521215f4a3 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 12:44:39 -0700 Subject: [PATCH 199/269] Update cloudbuild_playground_backend.yml --- gha-cloudbuild/cloudbuild_playground_backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index ef65ab2da114..f772c0a15f31 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -61,4 +61,4 @@ substitutions: images: - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -timeout: 1800s \ No newline at end of file +timeout: 180000s \ No newline at end of file From 2cff7e52be135e89d26401281489cbf1244a7643 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 25 Oct 2022 12:45:54 -0700 Subject: [PATCH 200/269] Update cloudbuild_playground_backend.yml --- gha-cloudbuild/cloudbuild_playground_backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index f772c0a15f31..1e700959f184 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -61,4 +61,4 @@ substitutions: images: - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -timeout: 180000s \ No newline at end of file +timeout: 3600s \ No newline at end of file From 6c25bafcf8430cd7ecdea496d047216eb976c66a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 26 Oct 2022 08:41:58 -0700 Subject: [PATCH 201/269] Update cloudbuild_playground_backend.yml --- .../cloudbuild_playground_backend.yml | 98 ++++++++++--------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index 1e700959f184..a57df39c33b2 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -14,51 +14,55 @@ # limitations under the License. steps: - # This step builds the base image for Beam Playground backend builds - - name: 'gcr.io/cloud-builders/docker' - args: - - 'build' - - '--tag=us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - - '-f' - - 'gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend' - - '.' - - # This step builds Beam Playground backend container for golang - - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - entrypoint: './gradlew' - args: - - ':playground:backend:containers:go:docker' - - # This step builds Beam Playground backend container for java - - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - entrypoint: './gradlew' - args: - - ':playground:backend:containers:java:docker' - - # This step builds Beam Playground backend container for java - - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - entrypoint: './gradlew' - args: - - ':playground:backend:containers:python:docker' - - - # This step builds Beam Playground backend container for java - - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - entrypoint: './gradlew' - args: - - ':playground:backend:containers:router:docker' - - # This step builds Beam Playground backend container for java - - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - entrypoint: './gradlew' - args: - - ':playground:backend:containers:scio:docker' - -substitutions: - _ARTIFACT_REGISTRY_REPOSITORY_ID: playground-repository - -images: - - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' - -timeout: 3600s \ No newline at end of file + - name: 'golang' + args: ['./gradlew playground:backend:precommit'] + # This step builds the base image for Beam Playground backend builds +# - name: 'gcr.io/cloud-builders/docker' +# args: +# - 'build' +# - '--tag=us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# - '-f' +# - 'gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend' +# - '.' +# +# # This step builds Beam Playground backend container for golang +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:go:docker' +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:java:docker' +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:python:docker' +# +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:router:docker' +# +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:scio:docker' +# +#substitutions: +# _DOCKERTAG: ${GITHUB_SHA} +# _ARTIFACT_REGISTRY_REPOSITORY_ID: playground-repository +# +#images: +# - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# +#timeout: 3600s \ No newline at end of file From 2a371d776d12e89e73b045c6b0788da8db7d4fb9 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 26 Oct 2022 08:44:23 -0700 Subject: [PATCH 202/269] Update cloudbuild_playground_backend.yml --- gha-cloudbuild/cloudbuild_playground_backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index a57df39c33b2..8b4129a259c7 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -16,7 +16,7 @@ steps: - name: 'golang' - args: ['./gradlew playground:backend:precommit'] + args: ['./gradlew ../playground:backend:precommit'] # This step builds the base image for Beam Playground backend builds # - name: 'gcr.io/cloud-builders/docker' # args: From 524791e5e4cbccb65872933df9578835e7b1fb60 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 26 Oct 2022 08:47:51 -0700 Subject: [PATCH 203/269] New cloudbuild yaml for backend --- cloudbuild_playground_backend.yml | 68 +++++++++++++++++++ .../cloudbuild_playground_backend.yml | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 cloudbuild_playground_backend.yml diff --git a/cloudbuild_playground_backend.yml b/cloudbuild_playground_backend.yml new file mode 100644 index 000000000000..a57df39c33b2 --- /dev/null +++ b/cloudbuild_playground_backend.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + + - name: 'golang' + args: ['./gradlew playground:backend:precommit'] + # This step builds the base image for Beam Playground backend builds +# - name: 'gcr.io/cloud-builders/docker' +# args: +# - 'build' +# - '--tag=us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# - '-f' +# - 'gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend' +# - '.' +# +# # This step builds Beam Playground backend container for golang +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:go:docker' +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:java:docker' +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:python:docker' +# +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:router:docker' +# +# +# # This step builds Beam Playground backend container for java +# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# entrypoint: './gradlew' +# args: +# - ':playground:backend:containers:scio:docker' +# +#substitutions: +# _DOCKERTAG: ${GITHUB_SHA} +# _ARTIFACT_REGISTRY_REPOSITORY_ID: playground-repository +# +#images: +# - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' +# +#timeout: 3600s \ No newline at end of file diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml index 8b4129a259c7..a57df39c33b2 100644 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ b/gha-cloudbuild/cloudbuild_playground_backend.yml @@ -16,7 +16,7 @@ steps: - name: 'golang' - args: ['./gradlew ../playground:backend:precommit'] + args: ['./gradlew playground:backend:precommit'] # This step builds the base image for Beam Playground backend builds # - name: 'gcr.io/cloud-builders/docker' # args: From 07c619691cdae07f551038c6edf3a5c32547da01 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 26 Oct 2022 08:51:37 -0700 Subject: [PATCH 204/269] Update cloudbuild_playground_backend.yml --- cloudbuild_playground_backend.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudbuild_playground_backend.yml b/cloudbuild_playground_backend.yml index a57df39c33b2..03f637b8aa89 100644 --- a/cloudbuild_playground_backend.yml +++ b/cloudbuild_playground_backend.yml @@ -15,8 +15,8 @@ steps: - - name: 'golang' - args: ['./gradlew playground:backend:precommit'] + - name: 'openjdk:11' + args: ['-c', './gradlew playground:backend:precommit'] # This step builds the base image for Beam Playground backend builds # - name: 'gcr.io/cloud-builders/docker' # args: From 22e23a7ec3abaaa6fc81f90ee6a45624e38f50a9 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 26 Oct 2022 08:54:27 -0700 Subject: [PATCH 205/269] Update cloudbuild_playground_backend.yml --- cloudbuild_playground_backend.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild_playground_backend.yml b/cloudbuild_playground_backend.yml index 03f637b8aa89..216198cd370b 100644 --- a/cloudbuild_playground_backend.yml +++ b/cloudbuild_playground_backend.yml @@ -16,6 +16,7 @@ steps: - name: 'openjdk:11' + entrypoint: 'bin/bash' args: ['-c', './gradlew playground:backend:precommit'] # This step builds the base image for Beam Playground backend builds # - name: 'gcr.io/cloud-builders/docker' From 17689049208c8ed72d5fdf9186991724faca192c Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 7 Nov 2022 17:58:57 +0500 Subject: [PATCH 206/269] Update for PR 280 --- learning/tour-of-beam/terraform-v2/README.md | 40 ++++++++++----- .../terraform-v2/backend_state.tf | 18 ++++++- learning/tour-of-beam/terraform-v2/main.tf | 17 +++++++ .../terraform-v2/modules/api_enable/main.tf | 49 ++++++++++--------- .../modules/api_enable/variables.tf | 17 +++++++ .../terraform-v2/modules/buckets/main.tf | 17 +++++++ .../terraform-v2/modules/buckets/outputs.tf | 17 +++++++ .../terraform-v2/modules/buckets/variables.tf | 17 +++++++ .../modules/cloud_functions/main.tf | 18 +++++++ .../modules/cloud_functions/outputs.tf | 17 +++++++ .../modules/cloud_functions/variables.tf | 17 +++++++ .../terraform-v2/modules/iam/main.tf | 19 ++++++- .../terraform-v2/modules/iam/outputs.tf | 17 +++++++ .../terraform-v2/modules/iam/variables.tf | 20 +++++++- learning/tour-of-beam/terraform-v2/outputs.tf | 17 +++++++ .../tour-of-beam/terraform-v2/variables.tf | 34 ++++++++++--- 16 files changed, 306 insertions(+), 45 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/README.md b/learning/tour-of-beam/terraform-v2/README.md index 17ac6b6eb7f8..fee4e71afe28 100644 --- a/learning/tour-of-beam/terraform-v2/README.md +++ b/learning/tour-of-beam/terraform-v2/README.md @@ -1,19 +1,36 @@ -# [Tour of Beam] Deploying backend infrastructure as GCP Cloud Function with GCP, Terraform and Cloud Build -This tutorial explains how to create Tour of Beam backend infrastructure as a GCP Cloud Functions using Infrastructure as a Code (IaC) and GitOps methodoliges. + + +# Overview + +This project deploys backend infrastructure for Tour of Beam as GCP Cloud Functions using Terraform. + +# Requirements + +## Development requirements + +- [Active GCP project with billing enabled](https://developers.google.com/workspace/guides/create-project) +- [Existing bucket to store Terraform state](https://cloud.google.com/storage/docs/creating-buckets) +- [Existing service account with the following roles](https://cloud.google.com/iam/docs/creating-managing-service-accounts): - Cloud Functions Admin - Project IAM Admin - Service Account Admin - Service Account User - Storage Admin - Storage Object Admin -- Exported JSON Key of service account above #### Configuring your environment @@ -28,13 +45,14 @@ Steps below will: # Create environment directory per your requirements/policy mkdir environments/dev cd ../environments/dev -# Import exported JSON Key using -export GOOGLE_APPLICATION_CREDENTIALS = '..key path...' +# Create new configuration to auth to GCP Project +gcloud init +# Acquire new user credentials to use for Application Default Credentials +gcloud auth application-default login # Initiliaze and run terraform terraform init terraform plan terraform apply -terraform destroy ``` diff --git a/learning/tour-of-beam/terraform-v2/backend_state.tf b/learning/tour-of-beam/terraform-v2/backend_state.tf index 85723beb13a8..41cb78351a45 100644 --- a/learning/tour-of-beam/terraform-v2/backend_state.tf +++ b/learning/tour-of-beam/terraform-v2/backend_state.tf @@ -1,6 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + terraform { backend "gcs" { - bucket = "tour-of-beam-backend-tfstate-bucket" prefix = "terraform-state" } required_providers { diff --git a/learning/tour-of-beam/terraform-v2/main.tf b/learning/tour-of-beam/terraform-v2/main.tf index da89eb863e1d..ed3021552211 100644 --- a/learning/tour-of-beam/terraform-v2/main.tf +++ b/learning/tour-of-beam/terraform-v2/main.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + provider "google" { project = var.project_id } diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf index a0fb23a12a95..d7ff91fffd9f 100644 --- a/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/api_enable/main.tf @@ -1,27 +1,28 @@ -# Enable API for Cloud Build -resource "google_project_service" "cloud_build" { - project = var.project_id - service = "cloudbuild.googleapis.com" - disable_on_destroy = false -} - -# Enable API for Cloud Function -resource "google_project_service" "cloud_function" { - project = var.project_id - service = "cloudfunctions.googleapis.com" - disable_on_destroy = false -} - -# Enable API for Resource Manager -resource "google_project_service" "resource_manager" { - project = var.project_id - service = "cloudresourcemanager.googleapis.com" - disable_on_destroy = false -} +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. -# Enable API for IAM -resource "google_project_service" "iam" { - project = var.project_id - service = "iam.googleapis.com" +resource "google_project_service" "required_services" { + project = var.project_id + for_each = toset([ + "cloudbuild", + "cloudfunctions", + "cloudresourcemanager", + "iam", + ]) + service = "${each.key}.googleapis.com" disable_on_destroy = false } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf b/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf index 7488e265382a..fef4fbca2c2b 100644 --- a/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/api_enable/variables.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + variable "project_id" { description = "The ID of the Google Cloud project within which resources are provisioned" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf index 85f100e4672d..580abf0655d5 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/main.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + resource "google_storage_bucket" "cloud_functions_bucket" { name = var.bucket_name location = var.location diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf index 4841dc829edb..e2f16108f54a 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/outputs.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + output "functions-bucket-id" { value = google_storage_bucket.cloud_functions_bucket.id } diff --git a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf index e3522f2c0c39..d60fee33091e 100644 --- a/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/buckets/variables.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + #Generates archive of source code variable "bucket_name" { description = "The bucket name to store functions' source code" diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf index 45ceb8c5f3b3..b0da7a27065b 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/main.tf @@ -1,3 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + resource "google_cloudfunctions_function" "cloud_function" { count = length(var.entry_point_names) name = "${var.entry_point_names[count.index]}" diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf index 72b116a897e8..b00710d1dda3 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/outputs.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + output "cloud-function-trigger-url" { value = google_cloudfunctions_function.cloud_function.*.https_trigger_url } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf index 189b192e08ef..abd7cdc955f2 100644 --- a/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/cloud_functions/variables.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + variable "region" {} variable "project_id" {} variable "service_account_id" { diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf index 148a78c4398e..6341e5e774d2 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/main.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/main.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + resource "google_service_account" "cloud_function_sa" { account_id = var.service_account_id display_name = "Service Account to run Cloud Functions" @@ -6,7 +23,7 @@ resource "google_service_account" "cloud_function_sa" { resource "google_project_iam_member" "terraform_service_account_roles" { for_each = toset([ "roles/cloudfunctions.admin", "roles/storage.objectViewer", - "roles/storage.objectCreator", "roles/iam.serviceAccountUser" + "roles/iam.serviceAccountUser" ]) role = each.key member = "serviceAccount:${google_service_account.cloud_function_sa.email}" diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf index 57285172fce6..5971958aa03b 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/outputs.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + output "service-account-email" { value = google_service_account.cloud_function_sa.email } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf index ac40f8352b60..1dcceda523bb 100644 --- a/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf +++ b/learning/tour-of-beam/terraform-v2/modules/iam/variables.tf @@ -1,8 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + variable "project_id" { description = "The ID of the Google Cloud project within which resources are provisioned" } variable "service_account_id" { description = "The ID of the service account responsible for running Google Cloud functions" -} - +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform-v2/outputs.tf b/learning/tour-of-beam/terraform-v2/outputs.tf index ca0b2b553155..a1e2ccb7583b 100644 --- a/learning/tour-of-beam/terraform-v2/outputs.tf +++ b/learning/tour-of-beam/terraform-v2/outputs.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + output "service-account-email" { value = module.iam.service-account-email } diff --git a/learning/tour-of-beam/terraform-v2/variables.tf b/learning/tour-of-beam/terraform-v2/variables.tf index 55e10baf164e..121c09076ef7 100644 --- a/learning/tour-of-beam/terraform-v2/variables.tf +++ b/learning/tour-of-beam/terraform-v2/variables.tf @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + variable "bucket_name" { description = "The bucket name that will store functions' source code" } @@ -7,17 +24,22 @@ variable "project_id" { description = "The ID of the Google Cloud project within which resources are provisioned" } -variable "iac_service_account_id" { - type = string - description = "The ID of the service account responsible for provisioning Google Cloud resources using Infrastructure-as-Code" - default = "terraform" -} - variable "service_account_id" { type = string description = "The ID of the service account responsible for running Google Cloud functions" } +variable "impersonated_account_id" { + description = "The ID of the impersonated service account that will run TF scripts to deploy infrastructure" +} + variable "region" { default = "us-central1" +} + +data "google_service_account_access_token" "default" { + provider = google.impersonation + target_service_account = local.terraform_service_account + scopes = ["userinfo-email", "cloud-platform"] + lifetime = "1200s" } \ No newline at end of file From 8ef3ef07f2016f839094c3303b4ce636b0de6e85 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Wed, 2 Nov 2022 05:39:04 -0700 Subject: [PATCH 207/269] Create cloudbuild_trigger.yml --- .github/workflows/cloudbuild_trigger.yml | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/cloudbuild_trigger.yml diff --git a/.github/workflows/cloudbuild_trigger.yml b/.github/workflows/cloudbuild_trigger.yml new file mode 100644 index 000000000000..f60b92947a2a --- /dev/null +++ b/.github/workflows/cloudbuild_trigger.yml @@ -0,0 +1,28 @@ +name: GCP Cloud Build Trigger via GHA + +on: + release: + types: [published] + +job: + job_id: + name: Trigger Google Cloud Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - id: 'auth' + uses: 'google-github-actions/auth@v0' + with: + credentials_json: '${{ secrets.GCP_CREDENTIALS }}' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v0' + + - name: 'Use gcloud CLI' + run: 'gcloud info' + + - name: 'Start Google Cloud Build trigger' + run: | + export CLOUDSDK_CORE_DISABLE_PROMPTS=1 + gcloud beta builds triggers run ${{ secrets.GCP_BUILD_STAGING_TRIGGER_ID }} --tag=${{ github.event.release.tag_name }} From e8da240690c5a394b40da2dbc115b944ad75bb20 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Wed, 2 Nov 2022 06:29:30 -0700 Subject: [PATCH 208/269] Update cloudbuild_trigger.yml --- .github/workflows/cloudbuild_trigger.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cloudbuild_trigger.yml b/.github/workflows/cloudbuild_trigger.yml index f60b92947a2a..265f06b50cce 100644 --- a/.github/workflows/cloudbuild_trigger.yml +++ b/.github/workflows/cloudbuild_trigger.yml @@ -4,7 +4,7 @@ on: release: types: [published] -job: +jobs: job_id: name: Trigger Google Cloud Build runs-on: ubuntu-latest From 2d3fb665b0c0802b3523a1420ca387404ec4a4d4 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Wed, 2 Nov 2022 06:32:20 -0700 Subject: [PATCH 209/269] Update cloudbuild_trigger.yml --- .github/workflows/cloudbuild_trigger.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cloudbuild_trigger.yml b/.github/workflows/cloudbuild_trigger.yml index 265f06b50cce..8b9bb8c4836b 100644 --- a/.github/workflows/cloudbuild_trigger.yml +++ b/.github/workflows/cloudbuild_trigger.yml @@ -1,6 +1,7 @@ name: GCP Cloud Build Trigger via GHA on: + workflow_dispatch: release: types: [published] From 3f294d4d67b2a2553f9393ee56bb05bbc2f01d71 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Wed, 2 Nov 2022 08:45:56 -0700 Subject: [PATCH 210/269] Update cloudbuild_trigger.yml --- .github/workflows/cloudbuild_trigger.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/cloudbuild_trigger.yml b/.github/workflows/cloudbuild_trigger.yml index 8b9bb8c4836b..0f39f038083d 100644 --- a/.github/workflows/cloudbuild_trigger.yml +++ b/.github/workflows/cloudbuild_trigger.yml @@ -11,6 +11,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: 'pip3 install grpcio' - id: 'auth' uses: 'google-github-actions/auth@v0' @@ -22,8 +28,16 @@ jobs: - name: 'Use gcloud CLI' run: 'gcloud info' + + - name: 'Install gcloud alpha' + run: 'gcloud components install alpha' - name: 'Start Google Cloud Build trigger' run: | export CLOUDSDK_CORE_DISABLE_PROMPTS=1 gcloud beta builds triggers run ${{ secrets.GCP_BUILD_STAGING_TRIGGER_ID }} --tag=${{ github.event.release.tag_name }} + + - name: 'Start Google Cloud Build logging tailing' + run: | + export CLOUDSDK_PYTHON_SITEPACKAGES=1 + gcloud alpha logging tail "resource.type=build and resource.labels.project_id=sandbox-playground-001" From d402c89b4f766b5db938a396bf43a78cef273a38 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Wed, 2 Nov 2022 08:54:33 -0700 Subject: [PATCH 211/269] Update cloudbuild_trigger.yml --- .github/workflows/cloudbuild_trigger.yml | 32 +++++++++++++++++------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cloudbuild_trigger.yml b/.github/workflows/cloudbuild_trigger.yml index 0f39f038083d..a17869649d2f 100644 --- a/.github/workflows/cloudbuild_trigger.yml +++ b/.github/workflows/cloudbuild_trigger.yml @@ -6,11 +6,33 @@ on: types: [published] jobs: - job_id: + run_cloud_build: name: Trigger Google Cloud Build runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + + - id: 'auth' + uses: 'google-github-actions/auth@v0' + with: + credentials_json: '${{ secrets.GCP_CREDENTIALS }}' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v0' + + - name: 'Use gcloud CLI' + run: 'gcloud info' + + - name: 'Start Google Cloud Build trigger' + run: | + export CLOUDSDK_CORE_DISABLE_PROMPTS=1 + gcloud beta builds triggers run ${{ secrets.GCP_BUILD_STAGING_TRIGGER_ID }} --tag=${{ github.event.release.tag_name }} + + logs_tailing: + name: Run logs tailing + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 - name: Set up Python 3.8 uses: actions/setup-python@v2 with: @@ -25,18 +47,10 @@ jobs: - name: 'Set up Cloud SDK' uses: 'google-github-actions/setup-gcloud@v0' - - - name: 'Use gcloud CLI' - run: 'gcloud info' - name: 'Install gcloud alpha' run: 'gcloud components install alpha' - - name: 'Start Google Cloud Build trigger' - run: | - export CLOUDSDK_CORE_DISABLE_PROMPTS=1 - gcloud beta builds triggers run ${{ secrets.GCP_BUILD_STAGING_TRIGGER_ID }} --tag=${{ github.event.release.tag_name }} - - name: 'Start Google Cloud Build logging tailing' run: | export CLOUDSDK_PYTHON_SITEPACKAGES=1 From 1d2a7b2d2d5fa1f3767ae3a0f752a9119e2be0b8 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Wed, 2 Nov 2022 09:20:42 -0700 Subject: [PATCH 212/269] Update cloudbuild_trigger.yml --- .github/workflows/cloudbuild_trigger.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cloudbuild_trigger.yml b/.github/workflows/cloudbuild_trigger.yml index a17869649d2f..1abdf330bc9c 100644 --- a/.github/workflows/cloudbuild_trigger.yml +++ b/.github/workflows/cloudbuild_trigger.yml @@ -26,7 +26,7 @@ jobs: - name: 'Start Google Cloud Build trigger' run: | export CLOUDSDK_CORE_DISABLE_PROMPTS=1 - gcloud beta builds triggers run ${{ secrets.GCP_BUILD_STAGING_TRIGGER_ID }} --tag=${{ github.event.release.tag_name }} + gcloud beta builds triggers run ${{ secrets.GCP_BUILD_STAGING_TRIGGER_ID }} logs_tailing: name: Run logs tailing @@ -54,4 +54,4 @@ jobs: - name: 'Start Google Cloud Build logging tailing' run: | export CLOUDSDK_PYTHON_SITEPACKAGES=1 - gcloud alpha logging tail "resource.type=build and resource.labels.project_id=sandbox-playground-001" + gcloud alpha logging tail "resource.type=build" From fc56907fd09c192f27f287890b9c95902108127f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Thu, 3 Nov 2022 02:07:51 -0700 Subject: [PATCH 213/269] Delete cloudbuild_playground_backend.yml --- .../cloudbuild_playground_backend.yml | 68 ------------------- 1 file changed, 68 deletions(-) delete mode 100644 gha-cloudbuild/cloudbuild_playground_backend.yml diff --git a/gha-cloudbuild/cloudbuild_playground_backend.yml b/gha-cloudbuild/cloudbuild_playground_backend.yml deleted file mode 100644 index a57df39c33b2..000000000000 --- a/gha-cloudbuild/cloudbuild_playground_backend.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -steps: - - - name: 'golang' - args: ['./gradlew playground:backend:precommit'] - # This step builds the base image for Beam Playground backend builds -# - name: 'gcr.io/cloud-builders/docker' -# args: -# - 'build' -# - '--tag=us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# - '-f' -# - 'gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend' -# - '.' -# -# # This step builds Beam Playground backend container for golang -# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# entrypoint: './gradlew' -# args: -# - ':playground:backend:containers:go:docker' -# -# # This step builds Beam Playground backend container for java -# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# entrypoint: './gradlew' -# args: -# - ':playground:backend:containers:java:docker' -# -# # This step builds Beam Playground backend container for java -# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# entrypoint: './gradlew' -# args: -# - ':playground:backend:containers:python:docker' -# -# -# # This step builds Beam Playground backend container for java -# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# entrypoint: './gradlew' -# args: -# - ':playground:backend:containers:router:docker' -# -# -# # This step builds Beam Playground backend container for java -# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# entrypoint: './gradlew' -# args: -# - ':playground:backend:containers:scio:docker' -# -#substitutions: -# _DOCKERTAG: ${GITHUB_SHA} -# _ARTIFACT_REGISTRY_REPOSITORY_ID: playground-repository -# -#images: -# - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# -#timeout: 3600s \ No newline at end of file From 9961422ea0b5237021831bf9b42db05ff2595b51 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Thu, 3 Nov 2022 02:08:11 -0700 Subject: [PATCH 214/269] Delete Dockerfile-base-image-backend --- .../dockerfiles/Dockerfile-base-image-backend | 41 ------------------- 1 file changed, 41 deletions(-) delete mode 100644 gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend diff --git a/gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend b/gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend deleted file mode 100644 index a0f935287a64..000000000000 --- a/gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend +++ /dev/null @@ -1,41 +0,0 @@ -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### - -FROM gcr.io/cloud-builders/gcloud - -ARG GO_VERSION=1.19.2 -ENV PATH="${PATH}:/usr/local/go/bin" - -RUN apt-get update >/dev/null - -# Install Docker -# See: https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script -RUN curl -fsSL https://get.docker.com -o get-docker.sh -RUN sh ./get-docker.sh >/dev/null - -# Install Go -# See: https://go.dev/doc/install -RUN curl -OL https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz -RUN sha256sum go$GO_VERSION.linux-amd64.tar.gz -RUN tar -C /usr/local -xvf go$GO_VERSION.linux-amd64.tar.gz -RUN rm go$GO_VERSION.linux-amd64.tar.gz -RUN go version - -# Install Java -RUN apt-get install openjdk-11-jdk -y >/dev/null -RUN java -version \ No newline at end of file From 07a9fdaf3726f4a69c2b44fbf639c165dadb8512 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Thu, 3 Nov 2022 02:08:33 -0700 Subject: [PATCH 215/269] Delete cloudbuild_trigger.yml --- .github/workflows/cloudbuild_trigger.yml | 57 ------------------------ 1 file changed, 57 deletions(-) delete mode 100644 .github/workflows/cloudbuild_trigger.yml diff --git a/.github/workflows/cloudbuild_trigger.yml b/.github/workflows/cloudbuild_trigger.yml deleted file mode 100644 index 1abdf330bc9c..000000000000 --- a/.github/workflows/cloudbuild_trigger.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: GCP Cloud Build Trigger via GHA - -on: - workflow_dispatch: - release: - types: [published] - -jobs: - run_cloud_build: - name: Trigger Google Cloud Build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - id: 'auth' - uses: 'google-github-actions/auth@v0' - with: - credentials_json: '${{ secrets.GCP_CREDENTIALS }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v0' - - - name: 'Use gcloud CLI' - run: 'gcloud info' - - - name: 'Start Google Cloud Build trigger' - run: | - export CLOUDSDK_CORE_DISABLE_PROMPTS=1 - gcloud beta builds triggers run ${{ secrets.GCP_BUILD_STAGING_TRIGGER_ID }} - - logs_tailing: - name: Run logs tailing - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install dependencies - run: 'pip3 install grpcio' - - - id: 'auth' - uses: 'google-github-actions/auth@v0' - with: - credentials_json: '${{ secrets.GCP_CREDENTIALS }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v0' - - - name: 'Install gcloud alpha' - run: 'gcloud components install alpha' - - - name: 'Start Google Cloud Build logging tailing' - run: | - export CLOUDSDK_PYTHON_SITEPACKAGES=1 - gcloud alpha logging tail "resource.type=build" From b0c9531d45a95e0ab0931962ce42a208efe06509 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan <114978215+ruslan-ikhsan@users.noreply.github.com> Date: Thu, 3 Nov 2022 02:09:01 -0700 Subject: [PATCH 216/269] Delete cloudbuild_playground_backend.yml --- cloudbuild_playground_backend.yml | 69 ------------------------------- 1 file changed, 69 deletions(-) delete mode 100644 cloudbuild_playground_backend.yml diff --git a/cloudbuild_playground_backend.yml b/cloudbuild_playground_backend.yml deleted file mode 100644 index 216198cd370b..000000000000 --- a/cloudbuild_playground_backend.yml +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -steps: - - - name: 'openjdk:11' - entrypoint: 'bin/bash' - args: ['-c', './gradlew playground:backend:precommit'] - # This step builds the base image for Beam Playground backend builds -# - name: 'gcr.io/cloud-builders/docker' -# args: -# - 'build' -# - '--tag=us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# - '-f' -# - 'gha-cloudbuild/dockerfiles/Dockerfile-base-image-backend' -# - '.' -# -# # This step builds Beam Playground backend container for golang -# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# entrypoint: './gradlew' -# args: -# - ':playground:backend:containers:go:docker' -# -# # This step builds Beam Playground backend container for java -# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# entrypoint: './gradlew' -# args: -# - ':playground:backend:containers:java:docker' -# -# # This step builds Beam Playground backend container for java -# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# entrypoint: './gradlew' -# args: -# - ':playground:backend:containers:python:docker' -# -# -# # This step builds Beam Playground backend container for java -# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# entrypoint: './gradlew' -# args: -# - ':playground:backend:containers:router:docker' -# -# -# # This step builds Beam Playground backend container for java -# - name: 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# entrypoint: './gradlew' -# args: -# - ':playground:backend:containers:scio:docker' -# -#substitutions: -# _DOCKERTAG: ${GITHUB_SHA} -# _ARTIFACT_REGISTRY_REPOSITORY_ID: playground-repository -# -#images: -# - 'us-central1-docker.pkg.dev/$PROJECT_ID/$_ARTIFACT_REGISTRY_REPOSITORY_ID/backend-builder:latest' -# -#timeout: 3600s \ No newline at end of file From 239884f1dbd12e3f13ecc1f6343b9d2871d1d592 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 7 Nov 2022 18:01:54 +0500 Subject: [PATCH 217/269] Update variables.tf --- learning/tour-of-beam/terraform-v2/variables.tf | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/variables.tf b/learning/tour-of-beam/terraform-v2/variables.tf index 121c09076ef7..8d52d1a7223b 100644 --- a/learning/tour-of-beam/terraform-v2/variables.tf +++ b/learning/tour-of-beam/terraform-v2/variables.tf @@ -29,17 +29,6 @@ variable "service_account_id" { description = "The ID of the service account responsible for running Google Cloud functions" } -variable "impersonated_account_id" { - description = "The ID of the impersonated service account that will run TF scripts to deploy infrastructure" -} - variable "region" { default = "us-central1" -} - -data "google_service_account_access_token" "default" { - provider = google.impersonation - target_service_account = local.terraform_service_account - scopes = ["userinfo-email", "cloud-platform"] - lifetime = "1200s" } \ No newline at end of file From 278814e0a34559cfd88ac75593da9f335ad4c7d5 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 8 Nov 2022 14:41:10 +0500 Subject: [PATCH 218/269] Update README.md --- learning/tour-of-beam/terraform-v2/README.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/learning/tour-of-beam/terraform-v2/README.md b/learning/tour-of-beam/terraform-v2/README.md index fee4e71afe28..3647759f0ced 100644 --- a/learning/tour-of-beam/terraform-v2/README.md +++ b/learning/tour-of-beam/terraform-v2/README.md @@ -1,19 +1,3 @@ - - # Overview This project deploys backend infrastructure for Tour of Beam as GCP Cloud Functions using Terraform. From 06f4b5e5750550fd81f12ca9ac853d038210ef40 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 8 Nov 2022 14:47:38 +0500 Subject: [PATCH 219/269] Update README.md --- learning/tour-of-beam/terraform-v2/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/learning/tour-of-beam/terraform-v2/README.md b/learning/tour-of-beam/terraform-v2/README.md index 3647759f0ced..71215f4408de 100644 --- a/learning/tour-of-beam/terraform-v2/README.md +++ b/learning/tour-of-beam/terraform-v2/README.md @@ -1,3 +1,20 @@ + + # Overview This project deploys backend infrastructure for Tour of Beam as GCP Cloud Functions using Terraform. From 91d07ac9d3bd4345fbbe5336e82f2191c70a0615 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 17:45:00 +0500 Subject: [PATCH 220/269] Create cloudbuild.yaml --- cloudbuild.yaml | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 cloudbuild.yaml diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 000000000000..a8e4962d8d8d --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +steps: + - name: 'ubuntu' + args: + - "-c" + - | + apt update > /dev/null + apt install -y software-properties-common + add-apt-repository -y ppa:deadsnakes/ppa && apt install python3.8 + cd playground/infrastructure + pip install -r requirements.txt + export \ + BEAM_ROOT_DIR="../../" \ + SDK_CONFIG="../../playground/sdks.yaml" \ + BEAM_EXAMPLE_CATEGORIES="../categories.yaml" \ + SERVER_ADDRESS="https://backend-${_SDK}-beta-dot-apache-beam-testing.appspot.com" \ + BEAM_USE_WEBGRPC=yes \ + BEAM_CONCURRENCY=4 + python3 ci_cd.py \ + --step ${_STEP} \ + --sdk SDK_${_SDK} \ + --origin ${_ORIGIN} \ + --subdirs ${_SUBDIRS} + done + +substitutions: + _STEP: CD + _SDK: JAVA + _ORIGIN: PG_EXAMPLES + _SUBDIRS: "../../learning/katas ../../examples ../../sdks" + +# This option enables writing logs to Cloud Logging +options: + logging: CLOUD_LOGGING_ONLY + +timeout: 3600s \ No newline at end of file From 456c456a3d36c54bbb084623c45e91daa99c5ecc Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 17:50:32 +0500 Subject: [PATCH 221/269] Update cloudbuild.yaml --- cloudbuild.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index a8e4962d8d8d..6c5491d73eb2 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -17,11 +17,12 @@ steps: - name: 'ubuntu' + entrypoint: 'bash' args: - - "-c" + - '-c' - | apt update > /dev/null - apt install -y software-properties-common + apt install -y software-properties-common > /dev/null add-apt-repository -y ppa:deadsnakes/ppa && apt install python3.8 cd playground/infrastructure pip install -r requirements.txt From 7a59a323b1e725cc357b1682a5b0177f663cc0da Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 18:04:29 +0500 Subject: [PATCH 222/269] Update cloudbuild.yaml --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 6c5491d73eb2..59932d25e0b4 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -23,7 +23,7 @@ steps: - | apt update > /dev/null apt install -y software-properties-common > /dev/null - add-apt-repository -y ppa:deadsnakes/ppa && apt install python3.8 + add-apt-repository -y ppa:deadsnakes/ppa && apt install python3.8 python3-pip cd playground/infrastructure pip install -r requirements.txt export \ From 4abdede32242f272f1bcef4c7a612722b51564e9 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 18:11:55 +0500 Subject: [PATCH 223/269] Update cloudbuild.yaml --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 59932d25e0b4..83613e2d9442 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -25,7 +25,7 @@ steps: apt install -y software-properties-common > /dev/null add-apt-repository -y ppa:deadsnakes/ppa && apt install python3.8 python3-pip cd playground/infrastructure - pip install -r requirements.txt + yes | pip install -r requirements.txt export \ BEAM_ROOT_DIR="../../" \ SDK_CONFIG="../../playground/sdks.yaml" \ From 66b67b42e2549fe20730984478862fd9309262d2 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 18:16:35 +0500 Subject: [PATCH 224/269] Update cloudbuild.yaml --- cloudbuild.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 83613e2d9442..b2dfada5ee26 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -23,9 +23,9 @@ steps: - | apt update > /dev/null apt install -y software-properties-common > /dev/null - add-apt-repository -y ppa:deadsnakes/ppa && apt install python3.8 python3-pip + add-apt-repository -y ppa:deadsnakes/ppa && apt install -y python3.8 cd playground/infrastructure - yes | pip install -r requirements.txt + pip install -r requirements.txt export \ BEAM_ROOT_DIR="../../" \ SDK_CONFIG="../../playground/sdks.yaml" \ From 90c45e5315d64c6eb34acad1341de02a6225e904 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 18:37:50 +0500 Subject: [PATCH 225/269] Update cloudbuild.yaml --- cloudbuild.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index b2dfada5ee26..b1b772644d0d 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -22,8 +22,7 @@ steps: - '-c' - | apt update > /dev/null - apt install -y software-properties-common > /dev/null - add-apt-repository -y ppa:deadsnakes/ppa && apt install -y python3.8 + apt install -y python3.8 cd playground/infrastructure pip install -r requirements.txt export \ From 9f5b2d2d5638c21c84bcd884f77658816bc8527b Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 18:40:47 +0500 Subject: [PATCH 226/269] Update cloudbuild.yaml --- cloudbuild.yaml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index b1b772644d0d..d1f5c0587f62 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -29,19 +29,23 @@ steps: BEAM_ROOT_DIR="../../" \ SDK_CONFIG="../../playground/sdks.yaml" \ BEAM_EXAMPLE_CATEGORIES="../categories.yaml" \ - SERVER_ADDRESS="https://backend-${_SDK}-beta-dot-apache-beam-testing.appspot.com" \ BEAM_USE_WEBGRPC=yes \ BEAM_CONCURRENCY=4 - python3 ci_cd.py \ - --step ${_STEP} \ - --sdk SDK_${_SDK} \ - --origin ${_ORIGIN} \ - --subdirs ${_SUBDIRS} + for sdk in go java python; do + export SDK=$sdk && + export SERVER_ADDRESS=https://${SDK}.playground-uat.xyz && + python3 ci_cd.py --step ${_STEP} --sdk SDK_${SDK^^} --origin $_ORIGIN \ + --subdirs ${_SUBDIRS} done +# python3 ci_cd.py \ +# --step ${_STEP} \ +# --sdk SDK_${_SDK} \ +# --origin ${_ORIGIN} \ +# --subdirs ${_SUBDIRS} +# done substitutions: _STEP: CD - _SDK: JAVA _ORIGIN: PG_EXAMPLES _SUBDIRS: "../../learning/katas ../../examples ../../sdks" From fd3f8664feee76e0fa617068e76870f66efb5e9a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 18:43:44 +0500 Subject: [PATCH 227/269] Update cloudbuild.yaml --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index d1f5c0587f62..c5b7c3509e32 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -33,7 +33,7 @@ steps: BEAM_CONCURRENCY=4 for sdk in go java python; do export SDK=$sdk && - export SERVER_ADDRESS=https://${SDK}.playground-uat.xyz && + export SERVER_ADDRESS=https://$SDK.playground-uat.xyz && python3 ci_cd.py --step ${_STEP} --sdk SDK_${SDK^^} --origin $_ORIGIN \ --subdirs ${_SUBDIRS} done From 83ffcd6e0c9acf386c900661fec1437f65dde6cc Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 18:45:25 +0500 Subject: [PATCH 228/269] Update cloudbuild.yaml --- cloudbuild.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index c5b7c3509e32..8b9ff2f416d2 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -32,9 +32,9 @@ steps: BEAM_USE_WEBGRPC=yes \ BEAM_CONCURRENCY=4 for sdk in go java python; do - export SDK=$sdk && - export SERVER_ADDRESS=https://$SDK.playground-uat.xyz && - python3 ci_cd.py --step ${_STEP} --sdk SDK_${SDK^^} --origin $_ORIGIN \ + export sdkname=$sdk && + export SERVER_ADDRESS=https://sdkname.playground-uat.xyz && + python3 ci_cd.py --step ${_STEP} --sdk SDK_${sdkname^^} --origin ${_ORIGIN} \ --subdirs ${_SUBDIRS} done # python3 ci_cd.py \ From 9b5c413343307bec4624b72b899883d4089d16de Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 18:48:03 +0500 Subject: [PATCH 229/269] Update cloudbuild.yaml --- cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 8b9ff2f416d2..26a2ab8c3ad3 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -22,6 +22,7 @@ steps: - '-c' - | apt update > /dev/null + apt install -y software-properties-common apt install -y python3.8 cd playground/infrastructure pip install -r requirements.txt From 799133c84ea62e78fa4e72fbd5ebb94286db2963 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 18:52:12 +0500 Subject: [PATCH 230/269] Update cloudbuild.yaml --- cloudbuild.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 26a2ab8c3ad3..7ea706c05aaa 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -23,10 +23,12 @@ steps: - | apt update > /dev/null apt install -y software-properties-common + add-apt-repository -y ppa:deadsnakes/ppa apt install -y python3.8 cd playground/infrastructure pip install -r requirements.txt export \ + DEBIAN_FRONTEND=noninteractive \ BEAM_ROOT_DIR="../../" \ SDK_CONFIG="../../playground/sdks.yaml" \ BEAM_EXAMPLE_CATEGORIES="../categories.yaml" \ From acd3320ab753fb540e7d0225fc338eb2f33e6969 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 18:55:27 +0500 Subject: [PATCH 231/269] Update cloudbuild.yaml --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 7ea706c05aaa..494cc680321e 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -22,13 +22,13 @@ steps: - '-c' - | apt update > /dev/null + export DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common add-apt-repository -y ppa:deadsnakes/ppa apt install -y python3.8 cd playground/infrastructure pip install -r requirements.txt export \ - DEBIAN_FRONTEND=noninteractive \ BEAM_ROOT_DIR="../../" \ SDK_CONFIG="../../playground/sdks.yaml" \ BEAM_EXAMPLE_CATEGORIES="../categories.yaml" \ From 9bad046b31f6daf1d49f164fb8f4e4ea24676083 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 19:01:33 +0500 Subject: [PATCH 232/269] Update cloudbuild.yaml --- cloudbuild.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 494cc680321e..7dad4746c438 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -24,10 +24,9 @@ steps: apt update > /dev/null export DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common - add-apt-repository -y ppa:deadsnakes/ppa - apt install -y python3.8 + add-apt-repository -y ppa:deadsnakes/ppa && apt install -y python3.8 cd playground/infrastructure - pip install -r requirements.txt + yes | pip install -r requirements.txt export \ BEAM_ROOT_DIR="../../" \ SDK_CONFIG="../../playground/sdks.yaml" \ From c12e0ef25bc39ea6dfa52fb673f887ca2946dcaf Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 19:08:45 +0500 Subject: [PATCH 233/269] Update cloudbuild.yaml --- cloudbuild.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 7dad4746c438..cf19f25527dc 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -21,12 +21,13 @@ steps: args: - '-c' - | - apt update > /dev/null + apt-get update export DEBIAN_FRONTEND=noninteractive - apt install -y software-properties-common - add-apt-repository -y ppa:deadsnakes/ppa && apt install -y python3.8 + apt-get install -y software-properties-common + add-apt-repository -y ppa:deadsnakes/ppa && apt update + apt install -y python3.8 cd playground/infrastructure - yes | pip install -r requirements.txt + pip install -r requirements.txt export \ BEAM_ROOT_DIR="../../" \ SDK_CONFIG="../../playground/sdks.yaml" \ From 7d7290005db64a64c854fc34f69d5fca406a17bd Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 19:11:34 +0500 Subject: [PATCH 234/269] Update cloudbuild.yaml --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index cf19f25527dc..2a5269ea74eb 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -25,7 +25,7 @@ steps: export DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common add-apt-repository -y ppa:deadsnakes/ppa && apt update - apt install -y python3.8 + apt install -y python3.8 python3-pip cd playground/infrastructure pip install -r requirements.txt export \ From e223cc45716aa0d661fb2cb8b3cd41f9d8c19bef Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 19:21:08 +0500 Subject: [PATCH 235/269] Update cloudbuild.yaml --- cloudbuild.yaml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 2a5269ea74eb..68e13bd1641f 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -32,23 +32,21 @@ steps: BEAM_ROOT_DIR="../../" \ SDK_CONFIG="../../playground/sdks.yaml" \ BEAM_EXAMPLE_CATEGORIES="../categories.yaml" \ + SERVER_ADDRESS="https://backend-${_SDK}-beta-dot-apache-beam-testing.appspot.com" \ BEAM_USE_WEBGRPC=yes \ BEAM_CONCURRENCY=4 - for sdk in go java python; do - export sdkname=$sdk && - export SERVER_ADDRESS=https://sdkname.playground-uat.xyz && - python3 ci_cd.py --step ${_STEP} --sdk SDK_${sdkname^^} --origin ${_ORIGIN} \ - --subdirs ${_SUBDIRS} - done -# python3 ci_cd.py \ -# --step ${_STEP} \ -# --sdk SDK_${_SDK} \ -# --origin ${_ORIGIN} \ -# --subdirs ${_SUBDIRS} -# done + python3 ci_cd.py --step ${_STEP} --sdk SDK_${_SDK} --origin ${_ORIGIN} \ + --subdirs ${_SUBDIRS} + + env: + - 'ORIGIN=${_ORIGIN}' + - 'SDK=${_SDK}' + - 'STEP=${_STEP}' + - 'SUBDIRS=${_SUBDIRS}' substitutions: _STEP: CD + _SDK: GO _ORIGIN: PG_EXAMPLES _SUBDIRS: "../../learning/katas ../../examples ../../sdks" From 8703895d7e449e1fa814e07effe7c183948f9bcc Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 29 Nov 2022 19:29:55 +0500 Subject: [PATCH 236/269] Update cloudbuild.yaml --- cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 68e13bd1641f..6091765fd627 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -43,6 +43,7 @@ steps: - 'SDK=${_SDK}' - 'STEP=${_STEP}' - 'SUBDIRS=${_SUBDIRS}' + - 'GOOGLE_CLOUD_PROJECT=$PROJECT_ID' substitutions: _STEP: CD From eb3d2e50ed54fce57d8c71f83087f3a8cd02ea00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 10:48:24 -0500 Subject: [PATCH 237/269] Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#24318) Bumps [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) from 1.13.2 to 1.13.3. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/mq/v1.13.2...service/mq/v1.13.3) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 4 ++-- sdks/go.sum | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index b9d0728c7110..5e5cc6ee7fbb 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -30,7 +30,7 @@ require ( cloud.google.com/go/storage v1.28.0 github.com/aws/aws-sdk-go-v2 v1.17.1 github.com/aws/aws-sdk-go-v2/config v1.18.2 - github.com/aws/aws-sdk-go-v2/credentials v1.13.2 + github.com/aws/aws-sdk-go-v2/credentials v1.13.3 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.41 github.com/aws/aws-sdk-go-v2/service/s3 v1.29.3 github.com/aws/smithy-go v1.13.4 @@ -92,7 +92,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.17.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/sdks/go.sum b/sdks/go.sum index 2c1d96beab14..97455598f945 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -154,8 +154,9 @@ github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/y github.com/aws/aws-sdk-go-v2/config v1.18.2 h1:tRhTb3xMZsB0gW0sXWpqs9FeIP8iQp5SvnvwiPXzHwo= github.com/aws/aws-sdk-go-v2/config v1.18.2/go.mod h1:9XVoZTdD8ICjrgI5ddb8j918q6lEZkFYpb7uohgvU6c= github.com/aws/aws-sdk-go-v2/credentials v1.3.1/go.mod h1:r0n73xwsIVagq8RsxmZbGSRQFj9As3je72C2WzUIToc= -github.com/aws/aws-sdk-go-v2/credentials v1.13.2 h1:F/v1w0XcFDZjL0bCdi9XWJenoPKjGbzljBhDKcryzEQ= github.com/aws/aws-sdk-go-v2/credentials v1.13.2/go.mod h1:eAT5aj/WJ2UDIA0IVNFc2byQLeD89SDEi4cjzH/MKoQ= +github.com/aws/aws-sdk-go-v2/credentials v1.13.3 h1:ur+FHdp4NbVIv/49bUjBW+FE7e57HOo03ELodttmagk= +github.com/aws/aws-sdk-go-v2/credentials v1.13.3/go.mod h1:/rOMmqYBcFfNbRPU0iN9IgGqD5+V2yp3iWNmIlz0wI4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0/go.mod h1:2LAuqPx1I6jNfaGDucWfA2zqQCYCOMCDHiCOciALyNw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= @@ -191,8 +192,9 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vbo github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod h1:q7o0j7d7HrJk/vr9uUt3BVRASvcU7gYZB9PUgPiByXg= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.4 h1:YNncBj5dVYd05i4ZQ+YicOotSXo0ufc9P8kTioi13EM= github.com/aws/aws-sdk-go-v2/service/sts v1.17.4/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 h1:60SJ4lhvn///8ygCzYy2l53bFW/Q15bVfyjyAWo6zuw= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.5/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= @@ -214,12 +216,9 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3k github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0 h1:t/LhUZLVitR1Ow2YOnduCsavhwFUklBMoGVYUCqmCqk= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= From 9557ed511bb22a2d539a7e60f01023b6359ebae0 Mon Sep 17 00:00:00 2001 From: bulat safiullin Date: Mon, 3 Oct 2022 17:36:46 +0600 Subject: [PATCH 238/269] [Website] update table text content overflow #23460 --- .../en/community/presentation-materials.md | 2 +- .../dsls/sql/calcite/lexical-structure.md | 2 ++ .../dsls/sql/zetasql/data-types.md | 24 ++++++++++++++++++- .../documentation/dsls/sql/zetasql/lexical.md | 10 +++++++- .../dsls/sql/zetasql/operators.md | 12 +++++++++- .../dsls/sql/zetasql/query-syntax.md | 12 +++++++++- .../community/table_with_icons.html | 2 ++ 7 files changed, 59 insertions(+), 5 deletions(-) diff --git a/website/www/site/content/en/community/presentation-materials.md b/website/www/site/content/en/community/presentation-materials.md index 46641deca329..d73d23488cea 100644 --- a/website/www/site/content/en/community/presentation-materials.md +++ b/website/www/site/content/en/community/presentation-materials.md @@ -22,7 +22,7 @@ limitations under the License. Are you interested in learning more about Apache Beam or giving a talk about Apache Beam? Excellent! The Apache Beam community has created this collection of materials to help you get started. The community periodically reviews these materials to ensure they are accurate and up-to-date. -
+
diff --git a/website/www/site/content/en/documentation/dsls/sql/calcite/lexical-structure.md b/website/www/site/content/en/documentation/dsls/sql/calcite/lexical-structure.md index 607d76e1df37..1c9bb0d0c5da 100644 --- a/website/www/site/content/en/documentation/dsls/sql/calcite/lexical-structure.md +++ b/website/www/site/content/en/documentation/dsls/sql/calcite/lexical-structure.md @@ -335,6 +335,7 @@ language, and have the following characteristics: Beam SQL has the following reserved keywords. +
@@ -989,6 +990,7 @@ ZONE
+
Terminating Semicolons ---------------------- diff --git a/website/www/site/content/en/documentation/dsls/sql/zetasql/data-types.md b/website/www/site/content/en/documentation/dsls/sql/zetasql/data-types.md index ebfc6bd8571b..43a0149107d8 100644 --- a/website/www/site/content/en/documentation/dsls/sql/zetasql/data-types.md +++ b/website/www/site/content/en/documentation/dsls/sql/zetasql/data-types.md @@ -27,6 +27,7 @@ Beam ZetaSQL supports standard SQL scalar data types as well as extensions inclu

The following table contains data type properties and the data types that each property applies to:

+
{{< table >}} @@ -94,6 +95,7 @@ Types for an explanation of join conditions.
{{< /table >}} +
## Numeric types @@ -103,6 +105,7 @@ Types for an explanation of join conditions.

Integers are numeric values that do not have fractional components.

+
{{< table >}} @@ -121,11 +124,13 @@ Types for an explanation of join conditions.
{{< /table >}} +
### Floating point type

Floating point values are approximate numeric values with fractional components.

+
{{< table >}} @@ -144,9 +149,11 @@ Types for an explanation of join conditions.
{{< /table >}} +
## Boolean type +
{{< table >}} @@ -164,9 +171,11 @@ Types for an explanation of join conditions.
{{< /table >}} +
## String type +
{{< table >}} @@ -183,6 +192,7 @@ Types for an explanation of join conditions.
{{< /table >}} +

Input STRING values must be UTF-8 encoded and output STRING values will be UTF-8 encoded. Alternate encodings like CESU-8 and Modified UTF-8 are not treated as @@ -196,6 +206,7 @@ characters.

## Bytes type +
{{< table >}} @@ -212,6 +223,7 @@ characters.

{{< /table >}} +

STRING and BYTES are separate types that cannot be used interchangeably. Casts between STRING and BYTES enforce that the bytes are encoded using UTF-8.

@@ -222,6 +234,7 @@ Caution: SQL has millisecond `TIMESTAMP` precision. If a `TIMESTAMP` field has sub-millisecond precision, SQL throws an `IllegalArgumentException`. +
{{< table >}} @@ -242,6 +255,7 @@ precision.
{{< /table >}} +

A timestamp represents an absolute point in time, independent of any time zone or convention such as Daylight Savings Time.

@@ -322,6 +336,7 @@ when there is a leap second.

## Array type +
{{< table >}} @@ -338,6 +353,7 @@ when there is a leap second.

{{< /table >}} +

An ARRAY is an ordered list of zero or more elements of non-ARRAY values. ARRAYs of ARRAYs are not allowed. Queries that would produce an ARRAY of @@ -355,6 +371,7 @@ an ARRAY cannot directly contain another ARRAY.

ARRAY<T>

Examples

+
{{< table >}} @@ -402,9 +419,11 @@ the two ARRAYs because ARRAYs cannot hold other ARRAYs directly.
{{< /table >}} +
## Struct type +
{{< table >}} @@ -422,6 +441,7 @@ the two ARRAYs because ARRAYs cannot hold other ARRAYs directly.
{{< /table >}} +
### Declaring a STRUCT type @@ -431,6 +451,7 @@ the elements of a STRUCT can be arbitrarily complex.

STRUCT<T>

Examples

+
{{< table >}} @@ -469,6 +490,7 @@ STRUCT<inner_array ARRAY<INT64>>
{{< /table >}} +
### Limited comparisons for STRUCT

STRUCTs can be directly compared using equality operators:

@@ -480,4 +502,4 @@ STRUCT<inner_array ARRAY<INT64>>

Notice, though, that these direct equality comparisons compare the fields of the STRUCT pairwise in ordinal order ignoring any field names. If instead you want to compare identically named fields of a STRUCT, you can compare the -individual fields directly.

\ No newline at end of file +individual fields directly.

diff --git a/website/www/site/content/en/documentation/dsls/sql/zetasql/lexical.md b/website/www/site/content/en/documentation/dsls/sql/zetasql/lexical.md index 868c467f11a6..51c7a85ef0f0 100644 --- a/website/www/site/content/en/documentation/dsls/sql/zetasql/lexical.md +++ b/website/www/site/content/en/documentation/dsls/sql/zetasql/lexical.md @@ -90,6 +90,7 @@ double (") quotation marks, or triple-quoted with groups o

Quoted literals:

+
{{< table >}} @@ -118,6 +119,7 @@ double (") quotation marks, or triple-quoted with groups o
{{< /table >}} +

Prefix characters (r, R, b, B) are optional for quoted or triple-quoted strings, and indicate that the string is a raw/regex string or a byte sequence, respectively. For example, b'abc' and b'''abc''' are both interpreted as type bytes. Prefix characters are case insensitive.

@@ -127,6 +129,7 @@ example, b'abc' and b'''abc''' are both interpreted as

The table below lists all valid escape sequences for representing non-alphanumeric characters in string literals. Any sequence not in this table produces an error.

+
{{< table >}} @@ -203,6 +206,7 @@ Any sequence not in this table produces an error.

{{< /table >}} +

@@ -341,6 +345,7 @@ TIMESTAMP '2014-09-27 12:30:00 America/Argentina/Buenos_Aires'

Beam SQL follows these rules for case sensitivity:

+
{{< table >}} @@ -399,6 +404,7 @@ TIMESTAMP '2014-09-27 12:30:00 America/Argentina/Buenos_Aires'
{{< /table >}} +

@@ -414,6 +420,7 @@ language, and have the following characteristics:

Beam SQL has the following reserved keywords.

+
{{< table >}} @@ -525,6 +532,7 @@ WITHIN
{{< /table >}} +

@@ -572,4 +580,4 @@ WHERE x = 3;

Comment includes all characters, including newlines, enclosed by the first occurrence of '/*' and the first subsequent occurrence of '*/'. Nested comments are not supported. The second example contains a nested comment that -renders the query invalid.

\ No newline at end of file +renders the query invalid.

diff --git a/website/www/site/content/en/documentation/dsls/sql/zetasql/operators.md b/website/www/site/content/en/documentation/dsls/sql/zetasql/operators.md index 40ef33807680..65ed01618595 100644 --- a/website/www/site/content/en/documentation/dsls/sql/zetasql/operators.md +++ b/website/www/site/content/en/documentation/dsls/sql/zetasql/operators.md @@ -30,6 +30,7 @@ Common conventions: The following table lists all supported operators from highest to lowest precedence. Precedence determines the order in which operators will be evaluated within a statement. +
{{< table >}} @@ -211,6 +212,7 @@ lowest precedence. Precedence determines the order in which operators will be ev
{{< /table >}} +
Operators with the same precedence are left associative. This means that those operators are grouped together starting from the left and moving right. For @@ -247,6 +249,7 @@ is recommended over: ## Element access operators +
{{< table >}} @@ -277,6 +280,7 @@ e.g.expression.fieldname1.fieldname2...
{{< /table >}} +
## Arithmetic operators @@ -379,6 +383,7 @@ Result types for Unary Minus: All logical operators allow only BOOL input. +
{{< table >}} @@ -410,6 +415,7 @@ is TRUE. Returns NULL otherwise.
{{< /table >}} +
## Comparison operators @@ -440,6 +446,7 @@ The following rules apply when comparing these data types: + `NULL`: The convention holds here: any operation with a `NULL` input returns `NULL`. +
{{< table >}} @@ -514,6 +521,7 @@ most common. X is evaluated only once.
{{< /table >}} +
When testing values that have a STRUCT data type for equality, it's possible that one or more fields are `NULL`. In such cases: @@ -561,6 +569,7 @@ IS operators return TRUE or FALSE for the condition they are testing. They never return `NULL`, even for `NULL` inputs. If NOT is present, the output BOOL value is inverted. +
{{< table >}} @@ -595,4 +604,5 @@ otherwise.
-{{< /table >}} \ No newline at end of file +{{< /table >}} +
diff --git a/website/www/site/content/en/documentation/dsls/sql/zetasql/query-syntax.md b/website/www/site/content/en/documentation/dsls/sql/zetasql/query-syntax.md index b209d85486c1..8e1aaed7ad5b 100644 --- a/website/www/site/content/en/documentation/dsls/sql/zetasql/query-syntax.md +++ b/website/www/site/content/en/documentation/dsls/sql/zetasql/query-syntax.md @@ -799,6 +799,7 @@ illustrate the behavior of different query clauses.

assigned to their school (SchoolID).

Table PlayerStats:

+
{{< table >}} @@ -837,6 +838,7 @@ assigned to their school (SchoolID).

{{< /table >}} +

The PlayerStats table includes a list of player names (LastName) and the unique ID assigned to the opponent they played in a given game (OpponentID) and the @@ -884,6 +886,7 @@ mascot for that school (Mascot).

ON Roster.SchoolID = TeamMascot.SchoolID;

Results:

+
{{< table >}} @@ -922,12 +925,14 @@ ON Roster.SchoolID = TeamMascot.SchoolID;
{{< /table >}} +

2) FULL [OUTER] JOIN

Example:

SELECT * FROM Roster FULL JOIN TeamMascot
 ON Roster.SchoolID = TeamMascot.SchoolID;
+
{{< table >}} @@ -978,6 +983,7 @@ ON Roster.SchoolID = TeamMascot.SchoolID;
{{< /table >}} +

3) LEFT [OUTER] JOIN

Example:

@@ -985,6 +991,7 @@ ON Roster.SchoolID = TeamMascot.SchoolID; ON Roster.SchoolID = TeamMascot.SchoolID;

Results:

+
{{< table >}} @@ -1029,6 +1036,7 @@ ON Roster.SchoolID = TeamMascot.SchoolID;
{{< /table >}} +

4) RIGHT [OUTER] JOIN

Example:

@@ -1036,6 +1044,7 @@ ON Roster.SchoolID = TeamMascot.SchoolID; ON Roster.SchoolID = TeamMascot.SchoolID;

Results:

+
{{< table >}} @@ -1080,6 +1089,7 @@ ON Roster.SchoolID = TeamMascot.SchoolID;
{{< /table >}} +
### GROUP BY clause

Example:

@@ -1247,4 +1257,4 @@ EXCEPT DISTINCT SELECT LastName FROM Roster;

Results:

-
(empty)
\ No newline at end of file +
(empty)
diff --git a/website/www/site/layouts/shortcodes/community/table_with_icons.html b/website/www/site/layouts/shortcodes/community/table_with_icons.html index d136493e2c1f..64ab0a928c2c 100644 --- a/website/www/site/layouts/shortcodes/community/table_with_icons.html +++ b/website/www/site/layouts/shortcodes/community/table_with_icons.html @@ -11,6 +11,7 @@ */}} {{ $data := index $.Site.Data .Site.Language.Lang (.Get 0) }} {{ range $list := $data }} +
@@ -50,4 +51,5 @@ {{ end }}
+
{{ end }} From 528faa15440f67fcc845728fd255b3661cf9fd45 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Wed, 23 Nov 2022 16:36:17 +0100 Subject: [PATCH 239/269] [Metrics] Add 'performance tests' tag to JMH dashboard (related to #22238) --- .../dashboards/perftests_metrics/Java_JMH_benchmarks.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.test-infra/metrics/grafana/dashboards/perftests_metrics/Java_JMH_benchmarks.json b/.test-infra/metrics/grafana/dashboards/perftests_metrics/Java_JMH_benchmarks.json index 5f08edecfd55..2606b4c38138 100644 --- a/.test-infra/metrics/grafana/dashboards/perftests_metrics/Java_JMH_benchmarks.json +++ b/.test-infra/metrics/grafana/dashboards/perftests_metrics/Java_JMH_benchmarks.json @@ -1610,7 +1610,9 @@ "refresh": "", "schemaVersion": 30, "style": "dark", - "tags": [], + "tags": [ + "performance tests" + ], "templating": { "list": [] }, @@ -1636,5 +1638,5 @@ "timezone": "", "title": "Java JMH benchmarks", "uid": "kllfR2vVk", - "version": 10 + "version": 11 } \ No newline at end of file From 206ea0c873bb478a9e27fcb5712b0ab536dd0036 Mon Sep 17 00:00:00 2001 From: Anand Inguva <34158215+AnandInguva@users.noreply.github.com> Date: Wed, 23 Nov 2022 11:23:41 -0500 Subject: [PATCH 240/269] Update apache beam installation in notebook (#24336) * Update apache beam installation in notebooks * Update examples/notebooks/beam-ml/run_inference_pytorch.ipynb * update outputs in code install * Fix syntax --- .../beam-ml/run_inference_pytorch.ipynb | 22 +----- ...inference_pytorch_tensorflow_sklearn.ipynb | 74 +------------------ .../beam-ml/run_inference_sklearn.ipynb | 52 +------------ 3 files changed, 7 insertions(+), 141 deletions(-) diff --git a/examples/notebooks/beam-ml/run_inference_pytorch.ipynb b/examples/notebooks/beam-ml/run_inference_pytorch.ipynb index 90948709a3aa..3afc6bad9890 100644 --- a/examples/notebooks/beam-ml/run_inference_pytorch.ipynb +++ b/examples/notebooks/beam-ml/run_inference_pytorch.ipynb @@ -79,11 +79,7 @@ { "cell_type": "code", "source": [ - "# Because of updates to the Google Cloud APIs, Apache Beam SDK versions 2.34.0 to 2.40.0 have some dependency conflicts. For more details, see the following Beam issue: https://github.com/apache/beam/issues/22218.\n", - "# This workaround installs the the Apache Beam SDK without getting stuck for long time. After this step, you might need to restart the runtime.\n", - "!pip install google-api-core --quiet\n", - "!pip install google-cloud-pubsub google-cloud-bigquery-storage --quiet\n", - "!pip install apache-beam[gcp,dataframe] --quiet" + "!pip install apache_beam[gcp,dataframe] --quiet" ], "metadata": { "id": "loxD-rOVchRn" @@ -101,19 +97,9 @@ }, "outputId": "09e0026a-cf8e-455c-9580-bfaef44683ce" }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.12.1+cu113)\n", - "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (4.1.1)\n" - ] - } - ], + "outputs": [], "source": [ - "%pip install torch" + "%pip install torch --quiet" ] }, { @@ -1009,4 +995,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb b/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb index 6c1e765920d0..3dac52f9d7a6 100644 --- a/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb +++ b/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb @@ -112,79 +112,7 @@ "id": "MRASwRTxY-2u", "outputId": "28760c59-c4dc-4486-dbd2-e7ac2c92c3b8" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Requirement already satisfied: pip in /usr/local/lib/python3.7/dist-packages (22.3)\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.23.1)\n", - "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n", - "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n", - "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n", - "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.1)\n", - "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n", - "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.13.1)\n", - "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.28.1)\n", - "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n", - "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n", - "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.9.0)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.1.1)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Collecting google-api-core==1.32\n", - " Using cached google_api_core-1.32.0-py2.py3-none-any.whl (93 kB)\n", - "Requirement already satisfied: protobuf<4.0.0dev,>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core==1.32) (3.20.3)\n", - "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from google-api-core==1.32) (2022.4)\n", - "Requirement already satisfied: google-auth<2.0dev,>=1.25.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core==1.32) (1.35.0)\n", - "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core==1.32) (1.56.4)\n", - "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core==1.32) (57.4.0)\n", - "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core==1.32) (1.15.0)\n", - "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.7/dist-packages (from google-api-core==1.32) (21.3)\n", - "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core==1.32) (2.28.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.25.0->google-api-core==1.32) (4.9)\n", - "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.25.0->google-api-core==1.32) (4.2.4)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2.0dev,>=1.25.0->google-api-core==1.32) (0.2.8)\n", - "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=14.3->google-api-core==1.32) (3.0.9)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.32) (2.10)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.32) (2022.9.24)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.32) (1.24.3)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.32) (2.1.1)\n", - "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=1.25.0->google-api-core==1.32) (0.4.8)\n", - "Installing collected packages: google-api-core\n", - " Attempting uninstall: google-api-core\n", - " Found existing installation: google-api-core 1.33.2\n", - " Uninstalling google-api-core-1.33.2:\n", - " Successfully uninstalled google-api-core-1.33.2\n", - "Successfully installed google-api-core-1.32.0\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0m" - ] - }, - { - "data": { - "application/vnd.colab-display-data+json": { - "pip_warning": { - "packages": [ - "google" - ] - } - } - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "!pip install --upgrade pip\n", "!pip install apache_beam[gcp]>=2.40.0\n", diff --git a/examples/notebooks/beam-ml/run_inference_sklearn.ipynb b/examples/notebooks/beam-ml/run_inference_sklearn.ipynb index 1e97bfe48d3e..9afcccc30f60 100644 --- a/examples/notebooks/beam-ml/run_inference_sklearn.ipynb +++ b/examples/notebooks/beam-ml/run_inference_sklearn.ipynb @@ -96,55 +96,7 @@ "outputId": "336e8afc-6716-41dd-a438-500353189c62" }, "execution_count": 1, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\u001b[K |████████████████████████████████| 236 kB 3.9 MB/s \n", - "\u001b[K |████████████████████████████████| 408 kB 58.2 MB/s \n", - "\u001b[K |████████████████████████████████| 47 kB 1.8 MB/s \n", - "\u001b[K |████████████████████████████████| 115 kB 70.8 MB/s \n", - "\u001b[K |████████████████████████████████| 4.7 MB 66.1 MB/s \n", - "\u001b[K |████████████████████████████████| 185 kB 35.6 MB/s \n", - "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", - "tensorflow 2.8.2+zzzcolab20220719082949 requires protobuf<3.20,>=3.9.2, but you have protobuf 4.21.6 which is incompatible.\n", - "tensorflow-metadata 1.10.0 requires protobuf<4,>=3.13, but you have protobuf 4.21.6 which is incompatible.\n", - "google-cloud-translate 1.5.0 requires google-api-core[grpc]<2.0.0dev,>=1.6.0, but you have google-api-core 2.10.1 which is incompatible.\n", - "google-cloud-language 1.2.0 requires google-api-core[grpc]<2.0.0dev,>=1.6.0, but you have google-api-core 2.10.1 which is incompatible.\n", - "google-cloud-firestore 1.7.0 requires google-api-core[grpc]<2.0.0dev,>=1.14.0, but you have google-api-core 2.10.1 which is incompatible.\n", - "google-cloud-datastore 1.8.0 requires google-api-core[grpc]<2.0.0dev,>=1.6.0, but you have google-api-core 2.10.1 which is incompatible.\n", - "google-cloud-core 1.0.3 requires google-api-core<2.0.0dev,>=1.14.0, but you have google-api-core 2.10.1 which is incompatible.\n", - "firebase-admin 4.4.0 requires google-api-core[grpc]<2.0.0dev,>=1.14.0; platform_python_implementation != \"PyPy\", but you have google-api-core 2.10.1 which is incompatible.\u001b[0m\n", - "\u001b[K |████████████████████████████████| 10.9 MB 4.0 MB/s \n", - "\u001b[K |████████████████████████████████| 1.0 MB 36.7 MB/s \n", - "\u001b[K |████████████████████████████████| 270 kB 69.3 MB/s \n", - "\u001b[K |████████████████████████████████| 2.4 MB 54.7 MB/s \n", - "\u001b[K |████████████████████████████████| 508 kB 69.0 MB/s \n", - "\u001b[K |████████████████████████████████| 62 kB 1.5 MB/s \n", - "\u001b[K |████████████████████████████████| 151 kB 71.4 MB/s \n", - "\u001b[K |████████████████████████████████| 83 kB 1.9 MB/s \n", - "\u001b[K |████████████████████████████████| 173 kB 72.1 MB/s \n", - "\u001b[K |████████████████████████████████| 124 kB 82.0 MB/s \n", - "\u001b[K |████████████████████████████████| 148 kB 55.6 MB/s \n", - "\u001b[K |████████████████████████████████| 267 kB 91.1 MB/s \n", - "\u001b[K |████████████████████████████████| 183 kB 69.6 MB/s \n", - "\u001b[K |████████████████████████████████| 255 kB 69.1 MB/s \n", - "\u001b[K |████████████████████████████████| 435 kB 65.7 MB/s \n", - "\u001b[K |████████████████████████████████| 180 kB 69.9 MB/s \n", - "\u001b[K |████████████████████████████████| 270 kB 74.1 MB/s \n", - "\u001b[K |████████████████████████████████| 134 kB 66.4 MB/s \n", - "\u001b[?25h Building wheel for dill (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Building wheel for google-apitools (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Building wheel for docopt (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", - "tensorflow 2.8.2+zzzcolab20220719082949 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.2 which is incompatible.\n", - "google-cloud-translate 1.5.0 requires google-api-core[grpc]<2.0.0dev,>=1.6.0, but you have google-api-core 2.10.1 which is incompatible.\n", - "google-cloud-firestore 1.7.0 requires google-api-core[grpc]<2.0.0dev,>=1.14.0, but you have google-api-core 2.10.1 which is incompatible.\n", - "firebase-admin 4.4.0 requires google-api-core[grpc]<2.0.0dev,>=1.14.0; platform_python_implementation != \"PyPy\", but you have google-api-core 2.10.1 which is incompatible.\u001b[0m\n" - ] - } - ] + "outputs": [] }, { "cell_type": "markdown", @@ -522,4 +474,4 @@ ] } ] -} \ No newline at end of file +} From 960b09bbe3bdffa0e42c1169beee107782ff02d3 Mon Sep 17 00:00:00 2001 From: Chamikara Jayalath Date: Wed, 23 Nov 2022 09:02:16 -0800 Subject: [PATCH 241/269] Adds GCP core dependency to the test expansion service (#24308) --- sdks/java/testing/expansion-service/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/java/testing/expansion-service/build.gradle b/sdks/java/testing/expansion-service/build.gradle index 953f2aef79e1..934b414d6a0a 100644 --- a/sdks/java/testing/expansion-service/build.gradle +++ b/sdks/java/testing/expansion-service/build.gradle @@ -30,6 +30,7 @@ dependencies { testImplementation project(path: ":sdks:java:core", configuration: "shadow") testImplementation project(":sdks:java:io:parquet") testImplementation project(":sdks:java:expansion-service") + testRuntimeOnly project(":sdks:java:extensions:google-cloud-platform-core") testRuntimeOnly library.java.hadoop_client } From 9e782b38ea8cea054fb35eb55cdaf7350bf670fe Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Wed, 23 Nov 2022 11:49:06 +0100 Subject: [PATCH 242/269] [Spark dataset runner] Fix translation to run in the evaluation thread as well so exceptions are handled properly (closes #24330) --- ...parkStructuredStreamingPipelineResult.java | 7 ++- .../SparkStructuredStreamingRunner.java | 58 ++++++++++--------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineResult.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineResult.java index 1392ae8f0c7f..404c2bf57043 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineResult.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineResult.java @@ -24,6 +24,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import javax.annotation.Nullable; import org.apache.beam.runners.spark.structuredstreaming.metrics.MetricsAccumulator; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; @@ -40,12 +41,12 @@ public class SparkStructuredStreamingPipelineResult implements PipelineResult { final Future pipelineExecution; - final Runnable onTerminalState; + @Nullable final Runnable onTerminalState; PipelineResult.State state; SparkStructuredStreamingPipelineResult( - final Future pipelineExecution, final Runnable onTerminalState) { + final Future pipelineExecution, @Nullable final Runnable onTerminalState) { this.pipelineExecution = pipelineExecution; this.onTerminalState = onTerminalState; // pipelineExecution is expected to have started executing eagerly. @@ -123,7 +124,7 @@ public PipelineResult.State cancel() throws IOException { private void offerNewState(State newState) { State oldState = this.state; this.state = newState; - if (!oldState.isTerminal() && newState.isTerminal()) { + if (!oldState.isTerminal() && newState.isTerminal() && onTerminalState != null) { try { onTerminalState.run(); } catch (Exception e) { diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java index 7d6f40ae9046..7fc96bad755d 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java @@ -18,10 +18,13 @@ package org.apache.beam.runners.spark.structuredstreaming; import static org.apache.beam.runners.spark.SparkCommonPipelineOptions.prepareFilesToStage; +import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; +import javax.annotation.Nullable; import org.apache.beam.runners.core.construction.SplittableParDo; import org.apache.beam.runners.core.construction.graph.ProjectionPushdownOptimizer; import org.apache.beam.runners.core.metrics.MetricsPusher; @@ -42,7 +45,6 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.options.PipelineOptionsValidator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.spark.SparkContext; import org.apache.spark.SparkEnv$; @@ -145,29 +147,22 @@ public SparkStructuredStreamingPipelineResult run(final Pipeline pipeline) { + " https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html\n" + " It is still experimental, its coverage of the Beam model is partial. ***"); + PipelineTranslator.detectStreamingMode(pipeline, options); + checkArgument(!options.isStreaming(), "Streaming is not supported."); + // clear state of Aggregators, Metrics and Watermarks if exists. AggregatorsAccumulator.clear(); MetricsAccumulator.clear(); - final EvaluationContext evaluationContext = translatePipeline(pipeline); + final SparkSession sparkSession = SparkSessionFactory.getOrCreateSession(options); + initAccumulators(sparkSession.sparkContext()); - final ExecutorService executorService = - Executors.newSingleThreadExecutor( - new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LocalSpark-thread").build()); final Future submissionFuture = - executorService.submit( - () -> { - // TODO initialise other services: checkpointing, metrics system, listeners, ... - evaluationContext.evaluate(); - }); - executorService.shutdown(); - - Runnable onTerminalState = - options.getUseActiveSparkSession() - ? () -> {} - : () -> evaluationContext.getSparkSession().stop(); - SparkStructuredStreamingPipelineResult result = - new SparkStructuredStreamingPipelineResult(submissionFuture, onTerminalState); + runAsync(() -> translatePipeline(sparkSession, pipeline).evaluate()); + + final SparkStructuredStreamingPipelineResult result = + new SparkStructuredStreamingPipelineResult( + submissionFuture, stopSparkSession(sparkSession, options.getUseActiveSparkSession())); if (options.getEnableSparkMetricSinks()) { registerMetricsSource(options.getAppName()); @@ -185,11 +180,7 @@ public SparkStructuredStreamingPipelineResult run(final Pipeline pipeline) { return result; } - private EvaluationContext translatePipeline(Pipeline pipeline) { - PipelineTranslator.detectStreamingMode(pipeline, options); - Preconditions.checkArgument( - !options.isStreaming(), "%s does not support streaming pipelines.", getClass().getName()); - + private EvaluationContext translatePipeline(SparkSession sparkSession, Pipeline pipeline) { // Default to using the primitive versions of Read.Bounded and Read.Unbounded for non-portable // execution. // TODO(https://github.com/apache/beam/issues/20530): Use SDF read as default when we address @@ -205,9 +196,6 @@ private EvaluationContext translatePipeline(Pipeline pipeline) { PipelineTranslator.replaceTransforms(pipeline, options); prepareFilesToStage(options); - final SparkSession sparkSession = SparkSessionFactory.getOrCreateSession(options); - initAccumulators(sparkSession.sparkContext()); - PipelineTranslator pipelineTranslator = new PipelineTranslatorBatch(); return pipelineTranslator.translate(pipeline, sparkSession, options); } @@ -228,9 +216,25 @@ private void registerMetricsSource(String appName) { } /** Init Metrics/Aggregators accumulators. This method is idempotent. */ - public static void initAccumulators(SparkContext sparkContext) { + private static void initAccumulators(SparkContext sparkContext) { // Init metrics accumulators MetricsAccumulator.init(sparkContext); AggregatorsAccumulator.init(sparkContext); } + + private static Future runAsync(Runnable task) { + ThreadFactory factory = + new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("SparkStructuredStreamingRunner-thread") + .build(); + ExecutorService execService = Executors.newSingleThreadExecutor(factory); + Future future = execService.submit(task); + execService.shutdown(); + return future; + } + + private static @Nullable Runnable stopSparkSession(SparkSession session, boolean isProvided) { + return !isProvided ? () -> session.stop() : null; + } } From d87eb543753936c66f51c41e19c4218d7cbe28ba Mon Sep 17 00:00:00 2001 From: Lukasz Cwik Date: Wed, 23 Nov 2022 09:26:47 -0800 Subject: [PATCH 243/269] Update dataflow containers to coincide with objsize 0.6.1 update (#24326) --- sdks/python/apache_beam/runners/dataflow/internal/names.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py b/sdks/python/apache_beam/runners/dataflow/internal/names.py index aa6de4f7085e..34ef156415c0 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/names.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py @@ -36,10 +36,10 @@ # Update this version to the next version whenever there is a change that will # require changes to legacy Dataflow worker execution environment. -BEAM_CONTAINER_VERSION = 'beam-master-20221021' +BEAM_CONTAINER_VERSION = 'beam-master-20221122' # Update this version to the next version whenever there is a change that # requires changes to SDK harness container or SDK harness launcher. -BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20221021' +BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20221122' DATAFLOW_CONTAINER_IMAGE_REPOSITORY = 'gcr.io/cloud-dataflow/v1beta3' From d60cfdc4e5c5f8aad5d20b880a7ba19fb5cf81e9 Mon Sep 17 00:00:00 2001 From: Anand Inguva <34158215+AnandInguva@users.noreply.github.com> Date: Wed, 23 Nov 2022 14:35:25 -0500 Subject: [PATCH 244/269] Add test configurations for deterministic outputs on Dataflow (#24325) * Add test configurations for deterministic outputs on Dataflow * Fix groovy syntax * Add machine config type to README * Fixup names and units for RunInference Benchmark tests * Change machine type to n1-standard-2 --- .../job_InferenceBenchmarkTests_Python.groovy | 30 ++++++++++++------- ...ython_ML_RunInference_Benchmark_Tests.json | 14 ++++----- .../testing/benchmarks/inference/README.md | 24 +++++++++++++-- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy b/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy index 8a34496b0a90..de1915ce990c 100644 --- a/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy +++ b/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy @@ -35,10 +35,13 @@ def loadTestConfigurations = { job_name : 'benchmark-tests-pytorch-imagenet-python' + now, project : 'apache-beam-testing', region : 'us-central1', + machine_type : 'n1-standard-2', + num_workers : 75, + disk_size_gb : 50, + autoscaling_algorithm : 'NONE', staging_location : 'gs://temp-storage-for-perf-tests/loadtests', temp_location : 'gs://temp-storage-for-perf-tests/loadtests', requirements_file : 'apache_beam/ml/inference/torch_tests_requirements.txt', - experiments : 'no_use_multiple_sdk_containers', publish_to_big_query : true, metrics_dataset : 'beam_run_inference', metrics_table : 'torch_inference_imagenet_results_resnet101', @@ -47,7 +50,7 @@ def loadTestConfigurations = { influx_db_name : InfluxDBCredentialsHelper.InfluxDBDatabaseName, influx_hostname : InfluxDBCredentialsHelper.InfluxDBHostUrl, pretrained_model_name : 'resnet101', - input_file : 'gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt', + input_file : 'gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt', model_state_dict_path : 'gs://apache-beam-ml/models/torchvision.models.resnet101.pth', output : 'gs://temp-storage-for-end-to-end-tests/torch/result_101' + now + '.txt' ] @@ -60,10 +63,13 @@ def loadTestConfigurations = { job_name : 'benchmark-tests-pytorch-imagenet-python' + now, project : 'apache-beam-testing', region : 'us-central1', + machine_type : 'n1-standard-2', + num_workers : 75, + disk_size_gb : 50, + autoscaling_algorithm : 'NONE', staging_location : 'gs://temp-storage-for-perf-tests/loadtests', temp_location : 'gs://temp-storage-for-perf-tests/loadtests', requirements_file : 'apache_beam/ml/inference/torch_tests_requirements.txt', - experiments : 'no_use_multiple_sdk_containers', publish_to_big_query : true, metrics_dataset : 'beam_run_inference', metrics_table : 'torch_inference_imagenet_results_resnet152', @@ -72,7 +78,7 @@ def loadTestConfigurations = { influx_db_name : InfluxDBCredentialsHelper.InfluxDBDatabaseName, influx_hostname : InfluxDBCredentialsHelper.InfluxDBHostUrl, pretrained_model_name : 'resnet152', - input_file : 'gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt', + input_file : 'gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt', model_state_dict_path : 'gs://apache-beam-ml/models/torchvision.models.resnet152.pth', output : 'gs://temp-storage-for-end-to-end-tests/torch/result_resnet152' + now + '.txt' ] @@ -86,11 +92,13 @@ def loadTestConfigurations = { job_name : 'benchmark-tests-pytorch-language-modeling-bert-base-uncased' + now, project : 'apache-beam-testing', region : 'us-central1', + machine_type : 'n1-standard-2', + num_workers : 250, + disk_size_gb : 50, + autoscaling_algorithm : 'NONE', staging_location : 'gs://temp-storage-for-perf-tests/loadtests', temp_location : 'gs://temp-storage-for-perf-tests/loadtests', requirements_file : 'apache_beam/ml/inference/torch_tests_requirements.txt', - pickle_library : 'cloudpickle', - experiments : 'no_use_multiple_sdk_containers', publish_to_big_query : true, metrics_dataset : 'beam_run_inference', metrics_table : 'torch_language_modeling_bert_base_uncased', @@ -98,7 +106,7 @@ def loadTestConfigurations = { influx_measurement : 'torch_language_modeling_bert_base_uncased', influx_db_name : InfluxDBCredentialsHelper.InfluxDBDatabaseName, influx_hostname : InfluxDBCredentialsHelper.InfluxDBHostUrl, - input_file : 'gs://apache-beam-ml/testing/inputs/sentences_50k.txt', + input_file : 'gs://apache-beam-ml/testing/inputs/sentences_50k.txt', bert_tokenizer : 'bert-base-uncased', model_state_dict_path : 'gs://apache-beam-ml/models/huggingface.BertForMaskedLM.bert-base-uncased.pth', output : 'gs://temp-storage-for-end-to-end-tests/torch/result_bert_base_uncased' + now + '.txt', @@ -112,11 +120,13 @@ def loadTestConfigurations = { job_name : 'benchmark-tests-pytorch-language-modeling-bert-large-cased' + now, project : 'apache-beam-testing', region : 'us-central1', + machine_type : 'n1-standard-2', + num_workers : 250, + disk_size_gb : 50, + autoscaling_algorithm : 'NONE', staging_location : 'gs://temp-storage-for-perf-tests/loadtests', temp_location : 'gs://temp-storage-for-perf-tests/loadtests', requirements_file : 'apache_beam/ml/inference/torch_tests_requirements.txt', - pickle_library : 'cloudpickle', - experiments : 'no_use_multiple_sdk_containers', publish_to_big_query : true, metrics_dataset : 'beam_run_inference', metrics_table : 'torch_language_modeling_bert_large_uncased', @@ -124,7 +134,7 @@ def loadTestConfigurations = { influx_measurement : 'torch_language_modeling_bert_large_uncased', influx_db_name : InfluxDBCredentialsHelper.InfluxDBDatabaseName, influx_hostname : InfluxDBCredentialsHelper.InfluxDBHostUrl, - input_file : 'gs://apache-beam-ml/testing/inputs/sentences_50k.txt', + input_file : 'gs://apache-beam-ml/testing/inputs/sentences_50k.txt', bert_tokenizer : 'bert-large-uncased', model_state_dict_path : 'gs://apache-beam-ml/models/huggingface.BertForMaskedLM.bert-large-uncased.pth', output : 'gs://temp-storage-for-end-to-end-tests/torch/result_bert_large_uncased' + now + '.txt' diff --git a/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_ML_RunInference_Benchmark_Tests.json b/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_ML_RunInference_Benchmark_Tests.json index a622897ff0fc..226cd67d5a6e 100644 --- a/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_ML_RunInference_Benchmark_Tests.json +++ b/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_ML_RunInference_Benchmark_Tests.json @@ -424,7 +424,7 @@ "steppedLine": false, "targets": [ { - "alias": "mean_load_model_latency_milli_seconds_resnet101", + "alias": "mean_inference_batch_latency_resnet101", "groupBy": [ { "params": [ @@ -462,7 +462,7 @@ "tags": [] }, { - "alias": "mean_load_model_latency_milli_seconds_resnet_152", + "alias": "mean_inference_batch_latency_resnet_152", "groupBy": [ { "params": [ @@ -593,7 +593,7 @@ "steppedLine": false, "targets": [ { - "alias": "$mean_inference_batch_latency_bert_base_uncased", + "alias": "mean_inference_batch_latency_bert_base_uncased", "groupBy": [ { "params": [ @@ -762,7 +762,7 @@ "steppedLine": false, "targets": [ { - "alias": "mean_load_model_latency_milli_seconds_resnet101", + "alias": "mean_load_model_latency_resnet101", "groupBy": [ { "params": [ @@ -800,7 +800,7 @@ "tags": [] }, { - "alias": "mean_load_model_latency_milli_seconds_resnet_152", + "alias": "mean_load_model_latency_resnet_152", "groupBy": [ { "params": [ @@ -931,7 +931,7 @@ "steppedLine": false, "targets": [ { - "alias": "mean_load_model_latency_milli_seconds_bert_base_uncased", + "alias": "mean_load_model_latency_bert_base_uncased", "groupBy": [ { "params": [ @@ -969,7 +969,7 @@ "tags": [] }, { - "alias": "mean_load_model_latency_milli_seconds_bert_large_uncased", + "alias": "mean_load_model_latency_bert_large_uncased", "groupBy": [ { "params": [ diff --git a/sdks/python/apache_beam/testing/benchmarks/inference/README.md b/sdks/python/apache_beam/testing/benchmarks/inference/README.md index 236365ab4989..9ef269c73a56 100644 --- a/sdks/python/apache_beam/testing/benchmarks/inference/README.md +++ b/sdks/python/apache_beam/testing/benchmarks/inference/README.md @@ -29,7 +29,7 @@ The Pytorch RunInference Image Classification 50K benchmark runs an [example image classification pipeline](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/examples/inference/pytorch_image_classification.py) using various different resnet image classification models (the benchmarks on [Beam's dashboard](http://s.apache.org/beam-community-metrics/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1) -display [resnet101](https://huggingface.co/microsoft/resnet-101) and [resnet152](https://huggingface.co/microsoft/resnet-152)) +display [resnet101](https://pytorch.org/vision/main/models/generated/torchvision.models.resnet101.html) and [resnet152](https://pytorch.org/vision/stable/models/generated/torchvision.models.resnet152.html)) against 50,000 example images from the OpenImage dataset. The benchmarks produce the following metrics: @@ -38,6 +38,16 @@ the following metrics: - Mean Load Model Latency - the average amount of time it takes to load a model. This is done once per DoFn instance on worker startup, so the cost is amortized across the pipeline. +Approximate size of the models used in the tests +* resnet101: 170.5 MB +* resnet152: 230.4 MB + +The above tests are configured to run using following configurations + * machine_type: n1-standard-2 + * num_workers: 75 + * autoscaling_algorithm: NONE + * disk_size_gb: 50 + ## Pytorch RunInference Language Modeling The Pytorch RunInference Language Modeling benchmark runs an @@ -50,4 +60,14 @@ the following metrics: - Mean Inference Requested Batch Size - the average batch size that RunInference groups the images into for batch prediction - Mean Inference Batch Latency - the average amount of time it takes to perform inference on a given batch of images - Mean Load Model Latency - the average amount of time it takes to load a model. This is done once per DoFn instance on worker -startup, so the cost is amortized across the pipeline. \ No newline at end of file +startup, so the cost is amortized across the pipeline. + +Approximate size of the models used in the tests +* bert-base-uncased: 417.7 MB +* bert-large-uncased: 1.2 GB + +The above tests are configured to run using following configurations + * machine_type: n1-standard-2 + * num_workers: 250 + * autoscaling_algorithm: NONE + * disk_size_gb: 75 From 2dec8245621ac4cb7d1c33efa6430a390728e5fd Mon Sep 17 00:00:00 2001 From: Chamikara Jayalath Date: Wed, 23 Nov 2022 16:52:50 -0800 Subject: [PATCH 245/269] Updates ExpansionService to support dynamically discovering and expanding SchemaTransforms (#23413) * Updates ExpansionService to support dynamically discovering and expanding SchemaTransforms * Fixing checker framework errors. * Address reviewer comments * Addressing reviewer comments * Addressing reviewer comments --- .../v1/beam_expansion_api.proto | 28 + .../pipeline/v1/external_transforms.proto | 17 + .../expansion/service/ExpansionService.java | 75 ++- ...pansionServiceSchemaTransformProvider.java | 144 ++++++ ...ionServiceSchemaTransformProviderTest.java | 486 ++++++++++++++++++ .../apache_beam/portability/common_urns.py | 1 + .../python/apache_beam/transforms/external.py | 158 ++++-- .../apache_beam/transforms/external_test.py | 30 ++ 8 files changed, 898 insertions(+), 41 deletions(-) create mode 100644 sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProvider.java create mode 100644 sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProviderTest.java diff --git a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto index f3ab890005de..568f9c877410 100644 --- a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto +++ b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto @@ -30,6 +30,7 @@ option java_package = "org.apache.beam.model.expansion.v1"; option java_outer_classname = "ExpansionApi"; import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto"; +import "org/apache/beam/model/pipeline/v1/schema.proto"; message ExpansionRequest { // Set of components needed to interpret the transform, or which @@ -72,7 +73,34 @@ message ExpansionResponse { string error = 10; } +message DiscoverSchemaTransformRequest { +} + +message SchemaTransformConfig { + // Config schema of the SchemaTransform + org.apache.beam.model.pipeline.v1.Schema config_schema = 1; + + // Names of input PCollections + repeated string input_pcollection_names = 2; + + // Names of output PCollections + repeated string output_pcollection_names = 3; +} + +message DiscoverSchemaTransformResponse { + // A mapping from SchemaTransform ID to schema transform config of discovered + // SchemaTransforms + map schema_transform_configs = 1; + + // If list of identifies are empty, this may contain an error. + string error = 2; +} + // Job Service for constructing pipelines service ExpansionService { rpc Expand (ExpansionRequest) returns (ExpansionResponse); + + //A RPC to discover already registered SchemaTransformProviders. + // See https://s.apache.org/easy-multi-language for more details. + rpc DiscoverSchemaTransform (DiscoverSchemaTransformRequest) returns (DiscoverSchemaTransformResponse); } diff --git a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/external_transforms.proto b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/external_transforms.proto index baff2c0436f5..18cd02e3942c 100644 --- a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/external_transforms.proto +++ b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/external_transforms.proto @@ -51,6 +51,11 @@ message ExpansionMethods { // Transform payload will be of type JavaClassLookupPayload. JAVA_CLASS_LOOKUP = 0 [(org.apache.beam.model.pipeline.v1.beam_urn) = "beam:expansion:payload:java_class_lookup:v1"]; + + // Expanding a SchemaTransform identified by the expansion service. + // Transform payload will be of type SchemaTransformPayload. + SCHEMA_TRANSFORM = 1 [(org.apache.beam.model.pipeline.v1.beam_urn) = + "beam:expansion:payload:schematransform:v1"]; } } @@ -106,4 +111,16 @@ message BuilderMethod { bytes payload = 3; } +message SchemaTransformPayload { + // The identifier of the SchemaTransform (typically a URN). + string identifier = 1; + + // The configuration schema of the SchemaTransform. + Schema configuration_schema = 2; + // The configuration of the SchemaTransform. + // Should be decodable via beam:coder:row:v1. + // The schema of the Row should be compatible with the schema of the + // SchemaTransform. + bytes configuration_row = 3; +} diff --git a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java index fed01d2576e6..221c40f79202 100644 --- a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java +++ b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java @@ -35,6 +35,9 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.beam.model.expansion.v1.ExpansionApi; +import org.apache.beam.model.expansion.v1.ExpansionApi.DiscoverSchemaTransformRequest; +import org.apache.beam.model.expansion.v1.ExpansionApi.DiscoverSchemaTransformResponse; +import org.apache.beam.model.expansion.v1.ExpansionApi.SchemaTransformConfig; import org.apache.beam.model.expansion.v1.ExpansionServiceGrpc; import org.apache.beam.model.pipeline.v1.ExternalTransforms.ExpansionMethods; import org.apache.beam.model.pipeline.v1.ExternalTransforms.ExternalConfigurationPayload; @@ -63,6 +66,7 @@ import org.apache.beam.sdk.schemas.SchemaCoder; import org.apache.beam.sdk.schemas.SchemaRegistry; import org.apache.beam.sdk.schemas.SchemaTranslation; +import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.transforms.ExternalTransformBuilder; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.SerializableFunction; @@ -436,6 +440,10 @@ private Map getRegisteredTransforms() { return registeredTransforms; } + private Iterable getRegisteredSchemaTransforms() { + return ExpansionServiceSchemaTransformProvider.of().getAllProviders(); + } + private Map loadRegisteredTransforms() { ImmutableMap.Builder registeredTransformsBuilder = ImmutableMap.builder(); @@ -500,6 +508,8 @@ private Map loadRegisteredTransforms() { pipelineOptions.as(ExpansionServiceOptions.class).getJavaClassLookupAllowlist(); assert allowList != null; transformProvider = new JavaClassLookupTransformProvider(allowList); + } else if (getUrn(ExpansionMethods.Enum.SCHEMA_TRANSFORM).equals(urn)) { + transformProvider = ExpansionServiceSchemaTransformProvider.of(); } else { transformProvider = getRegisteredTransforms().get(urn); if (transformProvider == null) { @@ -604,6 +614,42 @@ public void expand( } } + DiscoverSchemaTransformResponse discover(DiscoverSchemaTransformRequest request) { + ExpansionServiceSchemaTransformProvider transformProvider = + ExpansionServiceSchemaTransformProvider.of(); + DiscoverSchemaTransformResponse.Builder responseBuilder = + DiscoverSchemaTransformResponse.newBuilder(); + for (org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider provider : + transformProvider.getAllProviders()) { + SchemaTransformConfig.Builder schemaTransformConfigBuider = + SchemaTransformConfig.newBuilder(); + schemaTransformConfigBuider.setConfigSchema( + SchemaTranslation.schemaToProto(provider.configurationSchema(), true)); + schemaTransformConfigBuider.addAllInputPcollectionNames(provider.inputCollectionNames()); + schemaTransformConfigBuider.addAllOutputPcollectionNames(provider.outputCollectionNames()); + responseBuilder.putSchemaTransformConfigs( + provider.identifier(), schemaTransformConfigBuider.build()); + } + + return responseBuilder.build(); + } + + @Override + public void discoverSchemaTransform( + DiscoverSchemaTransformRequest request, + StreamObserver responseObserver) { + try { + responseObserver.onNext(discover(request)); + responseObserver.onCompleted(); + } catch (RuntimeException exn) { + responseObserver.onNext( + ExpansionApi.DiscoverSchemaTransformResponse.newBuilder() + .setError(Throwables.getStackTraceAsString(exn)) + .build()); + responseObserver.onCompleted(); + } + } + @Override public void close() throws Exception { // Nothing to do because the expansion service is stateless. @@ -618,9 +664,36 @@ public static void main(String[] args) throws Exception { @SuppressWarnings("nullness") ExpansionService service = new ExpansionService(Arrays.copyOfRange(args, 1, args.length)); + + StringBuilder registeredTransformsLog = new StringBuilder(); + boolean registeredTransformsFound = false; + registeredTransformsLog.append("\n"); + registeredTransformsLog.append("Registered transforms:"); + for (Map.Entry entry : service.getRegisteredTransforms().entrySet()) { - System.out.println("\t" + entry.getKey() + ": " + entry.getValue()); + registeredTransformsFound = true; + registeredTransformsLog.append("\n\t" + entry.getKey() + ": " + entry.getValue()); + } + + StringBuilder registeredSchemaTransformProvidersLog = new StringBuilder(); + boolean registeredSchemaTransformProvidersFound = false; + registeredSchemaTransformProvidersLog.append("\n"); + registeredSchemaTransformProvidersLog.append("Registered SchemaTransformProviders:"); + + for (SchemaTransformProvider provider : service.getRegisteredSchemaTransforms()) { + registeredSchemaTransformProvidersFound = true; + registeredSchemaTransformProvidersLog.append("\n\t" + provider.identifier()); + } + + if (registeredTransformsFound) { + System.out.println(registeredTransformsLog.toString()); + } + if (registeredSchemaTransformProvidersFound) { + System.out.println(registeredSchemaTransformProvidersLog.toString()); + } + if (!registeredTransformsFound && !registeredSchemaTransformProvidersFound) { + System.out.println("\nDid not find any registered transforms or SchemaTransforms.\n"); } Server server = diff --git a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProvider.java b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProvider.java new file mode 100644 index 000000000000..4657e0524025 --- /dev/null +++ b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProvider.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.expansion.service; + +import static org.apache.beam.runners.core.construction.BeamUrns.getUrn; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; +import org.apache.beam.model.pipeline.v1.ExternalTransforms.ExpansionMethods; +import org.apache.beam.model.pipeline.v1.ExternalTransforms.SchemaTransformPayload; +import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.expansion.service.ExpansionService.TransformProvider; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.SchemaTranslation; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionRowTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.checkerframework.checker.nullness.qual.Nullable; + +@SuppressWarnings({"rawtypes"}) +public class ExpansionServiceSchemaTransformProvider + implements TransformProvider { + + private Map + schemaTransformProviders = new HashMap<>(); + private static @Nullable ExpansionServiceSchemaTransformProvider transformProvider = null; + + private ExpansionServiceSchemaTransformProvider() { + try { + for (org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider schemaTransformProvider : + ServiceLoader.load( + org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider.class)) { + if (schemaTransformProviders.containsKey(schemaTransformProvider.identifier())) { + throw new IllegalArgumentException( + "Found multiple SchemaTransformProvider implementations with the same identifier " + + schemaTransformProvider.identifier()); + } + schemaTransformProviders.put(schemaTransformProvider.identifier(), schemaTransformProvider); + } + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + public static ExpansionServiceSchemaTransformProvider of() { + if (transformProvider == null) { + transformProvider = new ExpansionServiceSchemaTransformProvider(); + } + + return transformProvider; + } + + @Override + public PCollectionRowTuple createInput(Pipeline p, Map> inputs) { + PCollectionRowTuple inputRowTuple = PCollectionRowTuple.empty(p); + for (Map.Entry> entry : inputs.entrySet()) { + inputRowTuple = inputRowTuple.and(entry.getKey(), (PCollection) entry.getValue()); + } + return inputRowTuple; + } + + @Override + public Map> extractOutputs(PCollectionRowTuple output) { + ImmutableMap.Builder> pCollectionMap = ImmutableMap.builder(); + for (String key : output.getAll().keySet()) { + pCollectionMap.put(key, output.get(key)); + } + return pCollectionMap.build(); + } + + @Override + public PTransform getTransform(FunctionSpec spec) { + SchemaTransformPayload payload; + try { + payload = SchemaTransformPayload.parseFrom(spec.getPayload()); + String identifier = payload.getIdentifier(); + if (!schemaTransformProviders.containsKey(identifier)) { + throw new RuntimeException( + "Did not find a SchemaTransformProvider with the identifier " + identifier); + } + + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException( + "Invalid payload type for URN " + getUrn(ExpansionMethods.Enum.SCHEMA_TRANSFORM), e); + } + + String identifier = payload.getIdentifier(); + org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider provider = + schemaTransformProviders.get(identifier); + if (provider == null) { + throw new IllegalArgumentException( + "Could not find a SchemaTransform with identifier " + identifier); + } + + Schema configSchemaFromRequest = + SchemaTranslation.schemaFromProto((payload.getConfigurationSchema())); + Schema configSchemaFromProvider = provider.configurationSchema(); + + if (!configSchemaFromRequest.assignableTo(configSchemaFromProvider)) { + throw new IllegalArgumentException( + String.format( + "Config schema provided with the expansion request %s is not compatible with the " + + "config of the Schema transform %s.", + configSchemaFromRequest, configSchemaFromProvider)); + } + + Row configRow; + try { + configRow = + RowCoder.of(provider.configurationSchema()) + .decode(payload.getConfigurationRow().newInput()); + } catch (IOException e) { + throw new RuntimeException("Error decoding payload", e); + } + + return provider.from(configRow).buildTransform(); + } + + Iterable getAllProviders() { + return schemaTransformProviders.values(); + } +} diff --git a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProviderTest.java b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProviderTest.java new file mode 100644 index 000000000000..5b9b50b248a7 --- /dev/null +++ b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProviderTest.java @@ -0,0 +1,486 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.expansion.service; + +import static org.apache.beam.runners.core.construction.BeamUrns.getUrn; +import static org.junit.Assert.assertEquals; + +import com.google.auto.service.AutoService; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.model.expansion.v1.ExpansionApi; +import org.apache.beam.model.pipeline.v1.ExternalTransforms; +import org.apache.beam.model.pipeline.v1.ExternalTransforms.ExpansionMethods; +import org.apache.beam.model.pipeline.v1.RunnerApi; +import org.apache.beam.runners.core.construction.PTransformTranslation; +import org.apache.beam.runners.core.construction.ParDoTranslation; +import org.apache.beam.runners.core.construction.PipelineTranslation; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.schemas.JavaFieldSchema; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.Schema.Field; +import org.apache.beam.sdk.schemas.Schema.FieldType; +import org.apache.beam.sdk.schemas.SchemaCoder; +import org.apache.beam.sdk.schemas.SchemaTranslation; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaCreate; +import org.apache.beam.sdk.schemas.transforms.SchemaTransform; +import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; +import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.Impulse; +import org.apache.beam.sdk.transforms.InferableFunction; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionRowTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.junit.Test; + +/** Tests for {@link ExpansionServiceSchemaTransformProvider}. */ +@SuppressWarnings({ + "rawtypes" // TODO(https://github.com/apache/beam/issues/20447) +}) +public class ExpansionServiceSchemaTransformProviderTest { + + private static final String TEST_NAME = "TestName"; + + private static final String TEST_NAMESPACE = "namespace"; + + private static final Schema TEST_SCHEMATRANSFORM_CONFIG_SCHEMA = + Schema.of( + Field.of("str1", FieldType.STRING), + Field.of("str2", FieldType.STRING), + Field.of("int1", FieldType.INT32), + Field.of("int2", FieldType.INT32)); + + private ExpansionService expansionService = new ExpansionService(); + + @DefaultSchema(JavaFieldSchema.class) + public static class TestSchemaTransformConfiguration { + + public final String str1; + public final String str2; + public final Integer int1; + public final Integer int2; + + @SchemaCreate + public TestSchemaTransformConfiguration(String str1, String str2, Integer int1, Integer int2) { + this.str1 = str1; + this.str2 = str2; + this.int1 = int1; + this.int2 = int2; + } + } + + /** Registers a SchemaTransform. */ + @AutoService(SchemaTransformProvider.class) + public static class TestSchemaTransformProvider + extends TypedSchemaTransformProvider { + + @Override + protected Class configurationClass() { + return TestSchemaTransformConfiguration.class; + } + + @Override + protected SchemaTransform from(TestSchemaTransformConfiguration configuration) { + return new TestSchemaTransform( + configuration.str1, configuration.str2, configuration.int1, configuration.int2); + } + + @Override + public String identifier() { + return "dummy_id"; + } + + @Override + public List inputCollectionNames() { + return ImmutableList.of("input1"); + } + + @Override + public List outputCollectionNames() { + return ImmutableList.of("output1"); + } + } + + public static class TestSchemaTransform implements SchemaTransform { + + private String str1; + private String str2; + private Integer int1; + private Integer int2; + + public TestSchemaTransform(String str1, String str2, Integer int1, Integer int2) { + this.str1 = str1; + this.str2 = str2; + this.int1 = int1; + this.int2 = int2; + } + + @Override + public PTransform buildTransform() { + return new TestTransform(str1, str2, int1, int2); + } + } + + public static class TestDoFn extends DoFn { + + public String str1; + public String str2; + public int int1; + public int int2; + + public TestDoFn(String str1, String str2, Integer int1, Integer int2) { + this.str1 = str1; + this.str2 = str2; + this.int1 = int1; + this.int2 = int2; + } + + @ProcessElement + public void processElement(@Element String element, OutputReceiver receiver) { + receiver.output(element); + } + } + + public static class TestTransform extends PTransform { + + private String str1; + private String str2; + private Integer int1; + private Integer int2; + + public TestTransform(String str1, String str2, Integer int1, Integer int2) { + this.str1 = str1; + this.str2 = str2; + this.int1 = int1; + this.int2 = int2; + } + + @Override + public PCollectionRowTuple expand(PCollectionRowTuple input) { + PCollection outputPC = + input + .getAll() + .values() + .iterator() + .next() + .apply( + MapElements.via( + new InferableFunction() { + @Override + public String apply(Row input) throws Exception { + return input.getString("in_str"); + } + })) + .apply(ParDo.of(new TestDoFn(this.str1, this.str2, this.int1, this.int2))) + .apply( + MapElements.via( + new InferableFunction() { + @Override + public Row apply(String input) throws Exception { + return Row.withSchema(Schema.of(Field.of("out_str", FieldType.STRING))) + .withFieldValue("out_str", input) + .build(); + } + })) + .setRowSchema(Schema.of(Field.of("out_str", FieldType.STRING))); + return PCollectionRowTuple.of("output1", outputPC); + } + } + + /** Registers a SchemaTransform. */ + @AutoService(SchemaTransformProvider.class) + public static class TestSchemaTransformProviderMultiInputMultiOutput + extends TypedSchemaTransformProvider { + + @Override + protected Class configurationClass() { + return TestSchemaTransformConfiguration.class; + } + + @Override + protected SchemaTransform from(TestSchemaTransformConfiguration configuration) { + return new TestSchemaTransformMultiInputOutput( + configuration.str1, configuration.str2, configuration.int1, configuration.int2); + } + + @Override + public String identifier() { + return "dummy_id_multi_input_multi_output"; + } + + @Override + public List inputCollectionNames() { + return ImmutableList.of("input1", "input2"); + } + + @Override + public List outputCollectionNames() { + return ImmutableList.of("output1", "output2"); + } + } + + public static class TestSchemaTransformMultiInputOutput implements SchemaTransform { + + private String str1; + private String str2; + private Integer int1; + private Integer int2; + + public TestSchemaTransformMultiInputOutput( + String str1, String str2, Integer int1, Integer int2) { + this.str1 = str1; + this.str2 = str2; + this.int1 = int1; + this.int2 = int2; + } + + @Override + public PTransform buildTransform() { + return new TestTransformMultiInputMultiOutput(str1, str2, int1, int2); + } + } + + public static class TestTransformMultiInputMultiOutput + extends PTransform { + + private String str1; + private String str2; + private Integer int1; + private Integer int2; + + public TestTransformMultiInputMultiOutput( + String str1, String str2, Integer int1, Integer int2) { + this.str1 = str1; + this.str2 = str2; + this.int1 = int1; + this.int2 = int2; + } + + @Override + public PCollectionRowTuple expand(PCollectionRowTuple input) { + PCollection outputPC1 = + input + .get("input1") + .apply( + MapElements.via( + new InferableFunction() { + @Override + public String apply(Row input) throws Exception { + return input.getString("in_str"); + } + })) + .apply(ParDo.of(new TestDoFn(this.str1, this.str2, this.int1, this.int2))) + .apply( + MapElements.via( + new InferableFunction() { + @Override + public Row apply(String input) throws Exception { + return Row.withSchema(Schema.of(Field.of("out_str", FieldType.STRING))) + .withFieldValue("out_str", input) + .build(); + } + })) + .setRowSchema(Schema.of(Field.of("out_str", FieldType.STRING))); + PCollection outputPC2 = + input + .get("input2") + .apply( + MapElements.via( + new InferableFunction() { + @Override + public String apply(Row input) throws Exception { + return input.getString("in_str"); + } + })) + .apply(ParDo.of(new TestDoFn(this.str1, this.str2, this.int1, this.int2))) + .apply( + MapElements.via( + new InferableFunction() { + @Override + public Row apply(String input) throws Exception { + return Row.withSchema(Schema.of(Field.of("out_str", FieldType.STRING))) + .withFieldValue("out_str", input) + .build(); + } + })) + .setRowSchema(Schema.of(Field.of("out_str", FieldType.STRING))); + return PCollectionRowTuple.of("output1", outputPC1, "output2", outputPC2); + } + } + + @Test + public void testSchemaTransformDiscovery() { + ExpansionApi.DiscoverSchemaTransformRequest discoverRequest = + ExpansionApi.DiscoverSchemaTransformRequest.newBuilder().build(); + ExpansionApi.DiscoverSchemaTransformResponse response = + expansionService.discover(discoverRequest); + assertEquals(2, response.getSchemaTransformConfigsCount()); + } + + private void verifyLeafTransforms(ExpansionApi.ExpansionResponse response, int count) { + + int leafTransformCount = 0; + for (RunnerApi.PTransform transform : response.getComponents().getTransformsMap().values()) { + if (transform.getSpec().getUrn().equals(PTransformTranslation.PAR_DO_TRANSFORM_URN)) { + RunnerApi.ParDoPayload parDoPayload; + try { + parDoPayload = RunnerApi.ParDoPayload.parseFrom(transform.getSpec().getPayload()); + DoFn doFn = ParDoTranslation.getDoFn(parDoPayload); + if (!(doFn instanceof TestDoFn)) { + continue; + } + TestDoFn testDoFn = (TestDoFn) doFn; + assertEquals("aaa", testDoFn.str1); + assertEquals("bbb", testDoFn.str2); + assertEquals(111, testDoFn.int1); + assertEquals(222, testDoFn.int2); + leafTransformCount++; + } catch (InvalidProtocolBufferException exc) { + throw new RuntimeException(exc); + } + } + } + assertEquals(count, leafTransformCount); + } + + @Test + public void testSchemaTransformExpansion() { + Pipeline p = Pipeline.create(); + p.apply(Impulse.create()); + RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(p); + + String inputPcollId = + Iterables.getOnlyElement( + Iterables.getOnlyElement(pipelineProto.getComponents().getTransformsMap().values()) + .getOutputsMap() + .values()); + Row configRow = + Row.withSchema(TEST_SCHEMATRANSFORM_CONFIG_SCHEMA) + .withFieldValue("str1", "aaa") + .withFieldValue("str2", "bbb") + .withFieldValue("int1", 111) + .withFieldValue("int2", 222) + .build(); + + ByteStringOutputStream outputStream = new ByteStringOutputStream(); + try { + SchemaCoder.of(configRow.getSchema()).encode(configRow, outputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ExternalTransforms.SchemaTransformPayload payload = + ExternalTransforms.SchemaTransformPayload.newBuilder() + .setIdentifier("dummy_id") + .setConfigurationRow(outputStream.toByteString()) + .setConfigurationSchema( + SchemaTranslation.schemaToProto(TEST_SCHEMATRANSFORM_CONFIG_SCHEMA, true)) + .build(); + + ExpansionApi.ExpansionRequest request = + ExpansionApi.ExpansionRequest.newBuilder() + .setComponents(pipelineProto.getComponents()) + .setTransform( + RunnerApi.PTransform.newBuilder() + .setUniqueName(TEST_NAME) + .setSpec( + RunnerApi.FunctionSpec.newBuilder() + .setUrn(getUrn(ExpansionMethods.Enum.SCHEMA_TRANSFORM)) + .setPayload(payload.toByteString())) + .putInputs("input1", inputPcollId)) + .setNamespace(TEST_NAMESPACE) + .build(); + ExpansionApi.ExpansionResponse response = expansionService.expand(request); + RunnerApi.PTransform expandedTransform = response.getTransform(); + + assertEquals(3, expandedTransform.getSubtransformsCount()); + assertEquals(1, expandedTransform.getInputsCount()); + assertEquals(1, expandedTransform.getOutputsCount()); + verifyLeafTransforms(response, 1); + } + + @Test + public void testSchemaTransformExpansionMultiInputMultiOutput() { + Pipeline p = Pipeline.create(); + p.apply(Impulse.create()); + p.apply(Impulse.create()); + RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(p); + + List inputPcollIds = new ArrayList<>(); + for (RunnerApi.PTransform transform : + pipelineProto.getComponents().getTransformsMap().values()) { + inputPcollIds.add(Iterables.getOnlyElement(transform.getOutputsMap().values())); + } + assertEquals(2, inputPcollIds.size()); + + Row configRow = + Row.withSchema(TEST_SCHEMATRANSFORM_CONFIG_SCHEMA) + .withFieldValue("str1", "aaa") + .withFieldValue("str2", "bbb") + .withFieldValue("int1", 111) + .withFieldValue("int2", 222) + .build(); + + ByteStringOutputStream outputStream = new ByteStringOutputStream(); + try { + SchemaCoder.of(configRow.getSchema()).encode(configRow, outputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ExternalTransforms.SchemaTransformPayload payload = + ExternalTransforms.SchemaTransformPayload.newBuilder() + .setIdentifier("dummy_id_multi_input_multi_output") + .setConfigurationRow(outputStream.toByteString()) + .setConfigurationSchema( + SchemaTranslation.schemaToProto(TEST_SCHEMATRANSFORM_CONFIG_SCHEMA, true)) + .build(); + + ExpansionApi.ExpansionRequest request = + ExpansionApi.ExpansionRequest.newBuilder() + .setComponents(pipelineProto.getComponents()) + .setTransform( + RunnerApi.PTransform.newBuilder() + .setUniqueName(TEST_NAME) + .setSpec( + RunnerApi.FunctionSpec.newBuilder() + .setUrn(getUrn(ExpansionMethods.Enum.SCHEMA_TRANSFORM)) + .setPayload(payload.toByteString())) + .putInputs("input1", inputPcollIds.get(0)) + .putInputs("input2", inputPcollIds.get(1))) + .setNamespace(TEST_NAMESPACE) + .build(); + ExpansionApi.ExpansionResponse response = expansionService.expand(request); + RunnerApi.PTransform expandedTransform = response.getTransform(); + + assertEquals(6, expandedTransform.getSubtransformsCount()); + assertEquals(2, expandedTransform.getInputsCount()); + assertEquals(2, expandedTransform.getOutputsCount()); + verifyLeafTransforms(response, 2); + } +} diff --git a/sdks/python/apache_beam/portability/common_urns.py b/sdks/python/apache_beam/portability/common_urns.py index 3b47f1ab1e40..3799af5d2e1b 100644 --- a/sdks/python/apache_beam/portability/common_urns.py +++ b/sdks/python/apache_beam/portability/common_urns.py @@ -78,6 +78,7 @@ displayData = StandardDisplayData.DisplayData java_class_lookup = ExpansionMethods.Enum.JAVA_CLASS_LOOKUP +schematransform_based_expand = ExpansionMethods.Enum.SCHEMA_TRANSFORM decimal = LogicalTypes.Enum.DECIMAL micros_instant = LogicalTypes.Enum.MICROS_INSTANT diff --git a/sdks/python/apache_beam/transforms/external.py b/sdks/python/apache_beam/transforms/external.py index 58b5182593ea..7a51379a0e39 100644 --- a/sdks/python/apache_beam/transforms/external.py +++ b/sdks/python/apache_beam/transforms/external.py @@ -28,6 +28,7 @@ import logging import threading from collections import OrderedDict +from collections import namedtuple from typing import Dict import grpc @@ -104,6 +105,28 @@ def payload(self): """ return self.build().SerializeToString() + def _get_schema_proto_and_payload(self, **kwargs): + named_fields = [] + fields_to_values = OrderedDict() + + for key, value in kwargs.items(): + if not key: + raise ValueError('Parameter name cannot be empty') + if value is None: + raise ValueError( + 'Received value None for key %s. None values are currently not ' + 'supported' % key) + named_fields.append( + (key, convert_to_typing_type(instance_to_type(value)))) + fields_to_values[key] = value + + schema_proto = named_fields_to_schema(named_fields) + row = named_tuple_from_schema(schema_proto)(**fields_to_values) + schema = named_tuple_to_schema(type(row)) + + payload = RowCoder(schema).encode(row) + return (schema_proto, payload) + class SchemaBasedPayloadBuilder(PayloadBuilder): """ @@ -156,6 +179,20 @@ def _get_named_tuple_instance(self): return self._tuple_instance +class SchemaTransformPayloadBuilder(PayloadBuilder): + def __init__(self, identifier, **kwargs): + self._identifier = identifier + self._kwargs = kwargs + + def build(self): + schema_proto, payload = self._get_schema_proto_and_payload(**self._kwargs) + payload = external_transforms_pb2.SchemaTransformPayload( + identifier=self._identifier, + configuration_schema=schema_proto, + configuration_row=payload) + return payload + + class JavaClassLookupPayloadBuilder(PayloadBuilder): """ Builds a payload for directly instantiating a Java transform using a @@ -177,45 +214,26 @@ def __init__(self, class_name): self._constructor_param_kwargs = None self._builder_methods_and_params = OrderedDict() - def _get_schema_proto_and_payload(self, *args, **kwargs): - named_fields = [] - fields_to_values = OrderedDict() + def _args_to_named_fields(self, args): next_field_id = 0 + named_fields = OrderedDict() for value in args: if value is None: raise ValueError( 'Received value None. None values are currently not supported') - named_fields.append( - ((JavaClassLookupPayloadBuilder.IGNORED_ARG_FORMAT % next_field_id), - convert_to_typing_type(instance_to_type(value)))) - fields_to_values[( + named_fields[( JavaClassLookupPayloadBuilder.IGNORED_ARG_FORMAT % next_field_id)] = value next_field_id += 1 - for key, value in kwargs.items(): - if not key: - raise ValueError('Parameter name cannot be empty') - if value is None: - raise ValueError( - 'Received value None for key %s. None values are currently not ' - 'supported' % key) - named_fields.append( - (key, convert_to_typing_type(instance_to_type(value)))) - fields_to_values[key] = value - - schema_proto = named_fields_to_schema(named_fields) - row = named_tuple_from_schema(schema_proto)(**fields_to_values) - schema = named_tuple_to_schema(type(row)) - - payload = RowCoder(schema).encode(row) - return (schema_proto, payload) + return named_fields def build(self): - constructor_param_args = self._constructor_param_args or [] - constructor_param_kwargs = self._constructor_param_kwargs or {} + all_constructor_param_kwargs = self._args_to_named_fields( + self._constructor_param_args) + if self._constructor_param_kwargs: + all_constructor_param_kwargs.update(self._constructor_param_kwargs) constructor_schema, constructor_payload = ( - self._get_schema_proto_and_payload( - *constructor_param_args, **constructor_param_kwargs)) + self._get_schema_proto_and_payload(**all_constructor_param_kwargs)) payload = external_transforms_pb2.JavaClassLookupPayload( class_name=self._class_name, constructor_schema=constructor_schema, @@ -225,9 +243,12 @@ def build(self): for builder_method_name, params in self._builder_methods_and_params.items(): builder_method_args, builder_method_kwargs = params + all_builder_method_kwargs = self._args_to_named_fields( + builder_method_args) + if builder_method_kwargs: + all_builder_method_kwargs.update(builder_method_kwargs) builder_method_schema, builder_method_payload = ( - self._get_schema_proto_and_payload( - *builder_method_args, **builder_method_kwargs)) + self._get_schema_proto_and_payload(**all_builder_method_kwargs)) builder_method = external_transforms_pb2.BuilderMethod( name=builder_method_name, schema=builder_method_schema, @@ -289,6 +310,64 @@ def _has_constructor(self): self._constructor_param_kwargs) +# Information regarding a SchemaTransform available in an external SDK. +SchemaTransformsConfig = namedtuple( + 'SchemaTransformsConfig', + ['identifier', 'configuration_schema', 'inputs', 'outputs']) + + +class SchemaAwareExternalTransform(ptransform.PTransform): + """A proxy transform for SchemaTransforms implemented in external SDKs. + + This allows Python pipelines to directly use existing SchemaTransforms + available to the expansion service without adding additional code in external + SDKs. + + :param identifier: unique identifier of the SchemaTransform. + :param expansion_service: an expansion service to use. This should already be + available and the Schema-aware transforms to be used must already be + deployed. + :param classpath: (Optional) A list paths to additional jars to place on the + expansion service classpath. + :kwargs: field name to value mapping for configuring the schema transform. + keys map to the field names of the schema of the SchemaTransform + (in-order). + """ + def __init__(self, identifier, expansion_service, classpath=None, **kwargs): + self._expansion_service = expansion_service + self._payload_builder = SchemaTransformPayloadBuilder(identifier, **kwargs) + self._classpath = classpath + + def expand(self, pcolls): + # Expand the transform using the expansion service. + return pcolls | ExternalTransform( + common_urns.schematransform_based_expand.urn, + self._payload_builder, + self._expansion_service) + + @staticmethod + def discover(expansion_service): + """Discover all SchemaTransforms available to the given expansion service. + + :return: a list of SchemaTransformsConfigs that represent the discovered + SchemaTransforms. + """ + + with ExternalTransform.service(expansion_service) as service: + discover_response = service.DiscoverSchemaTransform( + beam_expansion_api_pb2.DiscoverSchemaTransformRequest()) + + for identifier in discover_response.schema_transform_configs: + proto_config = discover_response.schema_transform_configs[identifier] + schema = named_tuple_from_schema(proto_config.config_schema) + + yield SchemaTransformsConfig( + identifier=identifier, + configuration_schema=schema, + inputs=proto_config.input_pcollection_names, + outputs=proto_config.output_pcollection_names) + + class JavaExternalTransform(ptransform.PTransform): """A proxy for Java-implemented external transforms. @@ -520,7 +599,7 @@ def expand(self, pvalueish): transform=transform_proto, output_coder_requests=output_coders) - with self._service() as service: + with ExternalTransform.service(self._expansion_service) as service: response = service.Expand(request) if response.error: raise RuntimeError(response.error) @@ -549,9 +628,10 @@ def fix_output(pcoll, tag): return self._output_to_pvalueish(self._outputs) + @staticmethod @contextlib.contextmanager - def _service(self): - if isinstance(self._expansion_service, str): + def service(expansion_service): + if isinstance(expansion_service, str): channel_options = [("grpc.max_receive_message_length", -1), ("grpc.max_send_message_length", -1)] if hasattr(grpc, 'local_channel_credentials'): @@ -560,7 +640,7 @@ def _service(self): # TODO: update this to support secure non-local channels. channel_factory_fn = functools.partial( grpc.secure_channel, - self._expansion_service, + expansion_service, grpc.local_channel_credentials(), options=channel_options) else: @@ -568,15 +648,13 @@ def _service(self): # by older versions of grpc which may be pulled in due to other project # dependencies. channel_factory_fn = functools.partial( - grpc.insecure_channel, - self._expansion_service, - options=channel_options) + grpc.insecure_channel, expansion_service, options=channel_options) with channel_factory_fn() as channel: yield ExpansionAndArtifactRetrievalStub(channel) - elif hasattr(self._expansion_service, 'Expand'): - yield self._expansion_service + elif hasattr(expansion_service, 'Expand'): + yield expansion_service else: - with self._expansion_service as stub: + with expansion_service as stub: yield stub def _resolve_artifacts(self, components, service, dest): diff --git a/sdks/python/apache_beam/transforms/external_test.py b/sdks/python/apache_beam/transforms/external_test.py index c567f34330d8..f38876367c39 100644 --- a/sdks/python/apache_beam/transforms/external_test.py +++ b/sdks/python/apache_beam/transforms/external_test.py @@ -44,6 +44,7 @@ from apache_beam.transforms.external import JavaExternalTransform from apache_beam.transforms.external import JavaJarExpansionService from apache_beam.transforms.external import NamedTupleBasedPayloadBuilder +from apache_beam.transforms.external import SchemaTransformPayloadBuilder from apache_beam.typehints import typehints from apache_beam.typehints.native_type_compatibility import convert_to_beam_type from apache_beam.utils import proto_utils @@ -445,6 +446,35 @@ class DataclassTransform(beam.ExternalTransform): return get_payload(DataclassTransform(**values)) +class SchemaTransformPayloadBuilderTest(unittest.TestCase): + def test_build_payload(self): + ComplexType = typing.NamedTuple( + "ComplexType", [ + ("str_sub_field", str), + ("int_sub_field", int), + ]) + + payload_builder = SchemaTransformPayloadBuilder( + identifier='dummy_id', + str_field='aaa', + int_field=123, + object_field=ComplexType(str_sub_field="bbb", int_sub_field=456)) + payload_bytes = payload_builder.payload() + payload_from_bytes = proto_utils.parse_Bytes( + payload_bytes, external_transforms_pb2.SchemaTransformPayload) + + self.assertEqual('dummy_id', payload_from_bytes.identifier) + + expected_coder = RowCoder(payload_from_bytes.configuration_schema) + schema_transform_config = expected_coder.decode( + payload_from_bytes.configuration_row) + + self.assertEqual('aaa', schema_transform_config.str_field) + self.assertEqual(123, schema_transform_config.int_field) + self.assertEqual('bbb', schema_transform_config.object_field.str_sub_field) + self.assertEqual(456, schema_transform_config.object_field.int_sub_field) + + class JavaClassLookupPayloadBuilderTest(unittest.TestCase): def _verify_row(self, schema, row_payload, expected_values): row = RowCoder(schema).decode(row_payload) From 445ec8df14c0f2e16ea9df1147218a3546aa4514 Mon Sep 17 00:00:00 2001 From: Lukasz Cwik Date: Wed, 23 Nov 2022 22:28:00 -0800 Subject: [PATCH 246/269] Enable streaming runner v2 tests that were forgotten to be enabled. (#24321) * Enable streaming runner v2 tests that were forgotten to be enabled. These tests are run internally on Google and have been passing for quite some time. Fixes #19957 Fixes #20726 Fixes #20601 Fixes #20734 * Explicitly enable streaming engine for runner based autosharding test --- .../google-cloud-dataflow-java/build.gradle | 58 +++---------------- 1 file changed, 9 insertions(+), 49 deletions(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index b8f292df9f9d..6882a506ec91 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -166,8 +166,7 @@ def runnerV2PipelineOptions = [ "--region=${dataflowRegion}", "--tempRoot=${dataflowValidatesTempRoot}", "--sdkContainerImage=${dockerJavaImageContainer}:${dockerTag}", - // TODO(https://github.com/apache/beam/issues/20806) remove shuffle_mode=appliance with runner v2 once issue is resolved. - "--experiments=use_unified_worker,use_runner_v2,shuffle_mode=appliance", + "--experiments=use_unified_worker,use_runner_v2", "--firestoreDb=${firestoreDb}", ] @@ -388,7 +387,7 @@ task validatesRunnerStreaming { description "Validates Dataflow runner forcing streaming mode" dependsOn(createLegacyWorkerValidatesRunnerTest( name: 'validatesRunnerLegacyWorkerTestStreaming', - pipelineOptions: legacyPipelineOptions + ['--streaming=true'], + pipelineOptions: legacyPipelineOptions + ['--streaming'], excludedCategories: [ 'org.apache.beam.sdk.testing.UsesCommittedMetrics', 'org.apache.beam.sdk.testing.UsesMapState', @@ -463,6 +462,8 @@ task validatesRunnerV2 { description = "Runs the ValidatesRunner tests on Dataflow Runner V2" dependsOn(createRunnerV2ValidatesRunnerTest( name: 'validatesRunnerV2Test', + // TODO(https://github.com/apache/beam/issues/20806) remove shuffle_mode=appliance with runner v2 once issue is resolved. + pipelineOptions: runnerV2PipelineOptions + ['--experiments=shuffle_mode=appliance'], excludedCategories: [ 'org.apache.beam.sdk.testing.UsesOnWindowExpiration', 'org.apache.beam.sdk.testing.UsesStatefulParDo', @@ -473,6 +474,7 @@ task validatesRunnerV2 { excludedTests: [ 'org.apache.beam.sdk.transforms.ReshuffleTest.testReshuffleWithTimestampsStreaming', + // TODO(https://github.com/apache/beam/issues/18592): respect ParDo lifecycle. 'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testFnCallSequenceStateful', 'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundle', 'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundleStateful', @@ -500,55 +502,20 @@ task validatesRunnerV2Streaming { description = "Runs the ValidatesRunner tests on Dataflow Runner V2 forcing streaming mode" dependsOn(createRunnerV2ValidatesRunnerTest( name: 'validatesRunnerV2TestStreaming', - pipelineOptions: runnerV2PipelineOptions + ['--streaming=true'], + pipelineOptions: runnerV2PipelineOptions + ['--streaming', '--experiments=enable_streaming_engine'], excludedCategories: [ 'org.apache.beam.sdk.testing.LargeKeys$Above10KB', 'org.apache.beam.sdk.testing.UsesBoundedSplittableParDo', 'org.apache.beam.sdk.testing.UsesCommittedMetrics', - 'org.apache.beam.sdk.testing.UsesStrictTimerOrdering' /* https://github.com/apache/beam/issues/19957 */, - 'org.apache.beam.sdk.testing.UsesOnWindowExpiration', ], excludedTests: [ - 'org.apache.beam.sdk.transforms.windowing.WindowTest.testMergingCustomWindows', - 'org.apache.beam.sdk.transforms.windowing.WindowTest.testMergingCustomWindowsKeyedCollection', - 'org.apache.beam.sdk.transforms.windowing.WindowTest.testMergingCustomWindowsWithoutCustomWindowTypes', - 'org.apache.beam.examples.complete.TopWikipediaSessionsTest.testComputeTopUsers', - 'org.apache.beam.sdk.transforms.windowing.WindowingTest.testNonPartitioningWindowing', - - 'org.apache.beam.sdk.io.AvroIOTest.testWriteWindowed', - 'org.apache.beam.sdk.io.AvroIOTest.testWindowedAvroIOWriteViaSink', - 'org.apache.beam.sdk.extensions.sql.BeamSqlDslAggregationTest.testTriggeredTumble', + // TestStream only (partially) supported on UW 'org.apache.beam.sdk.transforms.ReshuffleTest.testReshuffleWithTimestampsStreaming', - // TODO(https://github.com/apache/beam/issues/20726) reading a side input twice fails - 'org.apache.beam.sdk.transforms.ParDoTest$MultipleInputsAndOutputTests.testSameSideInputReadTwice', - 'org.apache.beam.sdk.transforms.CombineFnsTest.testComposedCombineWithContext', - 'org.apache.beam.sdk.transforms.CombineTest$CombineWithContextTests.testSimpleCombineWithContextEmpty', - 'org.apache.beam.sdk.transforms.CombineTest$CombineWithContextTests.testSimpleCombineWithContext', - 'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testFixedWindowsCombineWithContext', - 'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testSessionsCombineWithContext', - 'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testSlidingWindowsCombineWithContext', - - 'org.apache.beam.runners.dataflow.DataflowRunnerTest.testBatchGroupIntoBatchesOverride', - + // TestStream with processing time control not supported + 'org.apache.beam.sdk.transforms.GroupByKeyTest$BasicTests.testAfterProcessingTimeContinuationTriggerUsingState', 'org.apache.beam.sdk.transforms.GroupByKeyTest.testCombiningAccumulatingProcessingTime', - // TODO(https://github.com/apache/beam/issues/20601): Pipeline is hanging for these 3 tests. - 'org.apache.beam.sdk.transforms.SplittableDoFnTest.testPairWithIndexBasicUnbounded', - 'org.apache.beam.sdk.transforms.SplittableDoFnTest.testOutputAfterCheckpointUnbounded', - 'org.apache.beam.sdk.transforms.SplittableDoFnTest.testBundleFinalizationOccursOnUnboundedSplittableDoFn', - - 'org.apache.beam.sdk.transforms.ParDoTest$TimerTests.testEventTimeTimerAlignAfterGcTimeUnbounded', - - 'org.apache.beam.sdk.metrics.MetricsTest$CommittedMetricTests.testCommittedCounterMetrics', - - 'org.apache.beam.sdk.transforms.WaitTest.testWaitWithSameFixedWindows', - 'org.apache.beam.sdk.transforms.WaitTest.testWaitWithDifferentFixedWindows', - 'org.apache.beam.sdk.transforms.WaitTest.testWaitWithSignalInSlidingWindows', - 'org.apache.beam.sdk.transforms.WaitTest.testWaitInGlobalWindow', - 'org.apache.beam.sdk.transforms.WaitTest.testWaitBoundedInDefaultWindow', - 'org.apache.beam.sdk.transforms.WaitTest.testWaitWithSomeSignalWindowsEmpty', - // TODO(https://github.com/apache/beam/issues/18592): respect ParDo lifecycle. 'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundle', 'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundleStateful', @@ -559,13 +526,6 @@ task validatesRunnerV2Streaming { 'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundle', 'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundleStateful', - // TODO(https://github.com/apache/beam/issues/20734) Empty flatten fails in streaming - "org.apache.beam.sdk.transforms.FlattenTest.testEmptyFlattenAsSideInput", - "org.apache.beam.sdk.transforms.FlattenTest.testFlattenPCollectionsEmptyThenParDo", - "org.apache.beam.sdk.transforms.FlattenTest.testFlattenPCollectionsEmpty", - - 'org.apache.beam.sdk.io.CountingSourceTest.testBoundedSourceSplits', - // TODO(https://github.com/apache/beam/issues/20931): Identify whether it's bug or a feature gap. 'org.apache.beam.sdk.transforms.GroupByKeyTest$WindowTests.testRewindowWithTimestampCombiner', From 62af16ec629d92dfe9cbd9a5b1e89c5297c24d41 Mon Sep 17 00:00:00 2001 From: Pablo Date: Wed, 23 Nov 2022 22:36:46 -0800 Subject: [PATCH 247/269] A schema transform implementation for SpannerIO.Write (#24278) * A schema transform implementation for SpannerIO.Write * fixup * fixup * fixup * fixup * fixup and comments * fixup * fixup --- .../sdk/io/gcp/spanner/MutationUtils.java | 2 +- .../SpannerWriteSchemaTransformProvider.java | 177 ++++++++++++++++++ .../sdk/io/gcp/spanner/SpannerWriteIT.java | 40 ++++ 3 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java index 085e9929a7bd..9b74e21d2852 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java @@ -88,7 +88,7 @@ private static Key createKeyFromBeamRow(Row row) { return builder.build(); } - private static Mutation createMutationFromBeamRows( + public static Mutation createMutationFromBeamRows( Mutation.WriteBuilder mutationBuilder, Row row) { Schema schema = row.getSchema(); List columns = schema.getFieldNames(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java new file mode 100644 index 000000000000..3e3462d8df0b --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.spanner; + +import com.google.auto.service.AutoService; +import com.google.auto.value.AutoValue; +import com.google.cloud.spanner.Mutation; +import java.io.Serializable; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.transforms.SchemaTransform; +import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; +import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; +import org.apache.beam.sdk.transforms.FlatMapElements; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.SimpleFunction; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionRowTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.checkerframework.checker.initialization.qual.Initialized; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.UnknownKeyFor; + +@AutoService(SchemaTransformProvider.class) +public class SpannerWriteSchemaTransformProvider + extends TypedSchemaTransformProvider< + SpannerWriteSchemaTransformProvider.SpannerWriteSchemaTransformConfiguration> { + + @Override + protected @UnknownKeyFor @NonNull @Initialized Class + configurationClass() { + return SpannerWriteSchemaTransformConfiguration.class; + } + + @Override + protected @UnknownKeyFor @NonNull @Initialized SchemaTransform from( + SpannerWriteSchemaTransformConfiguration configuration) { + return new SpannerSchemaTransformWrite(configuration); + } + + static class SpannerSchemaTransformWrite implements SchemaTransform, Serializable { + private final SpannerWriteSchemaTransformConfiguration configuration; + + SpannerSchemaTransformWrite(SpannerWriteSchemaTransformConfiguration configuration) { + this.configuration = configuration; + } + + @Override + public @UnknownKeyFor @NonNull @Initialized PTransform< + @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple, + @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple> + buildTransform() { + // TODO: For now we are allowing ourselves to fail at runtime, but we could + // perform validations here at expansion time. This TODO is to add a few + // validations (e.g. table/database/instance existence, schema match, etc). + return new PTransform<@NonNull PCollectionRowTuple, @NonNull PCollectionRowTuple>() { + @Override + public PCollectionRowTuple expand(@NonNull PCollectionRowTuple input) { + SpannerWriteResult result = + input + .get("input") + .apply( + MapElements.via( + new SimpleFunction( + row -> + MutationUtils.createMutationFromBeamRows( + Mutation.newInsertOrUpdateBuilder(configuration.getTableId()), + Objects.requireNonNull(row))) {})) + .apply( + SpannerIO.write() + .withDatabaseId(configuration.getDatabaseId()) + .withInstanceId(configuration.getInstanceId()) + .withFailureMode(SpannerIO.FailureMode.REPORT_FAILURES)); + Schema failureSchema = + Schema.builder() + .addStringField("operation") + .addStringField("instanceId") + .addStringField("databaseId") + .addStringField("tableId") + .addStringField("mutationData") + .build(); + PCollection failures = + result + .getFailedMutations() + .apply( + FlatMapElements.into(TypeDescriptors.rows()) + .via( + mtg -> + Objects.requireNonNull(mtg).attached().stream() + .map( + mutation -> + Row.withSchema(failureSchema) + .addValue(mutation.getOperation().toString()) + .addValue(configuration.getInstanceId()) + .addValue(configuration.getDatabaseId()) + .addValue(mutation.getTable()) + // TODO(pabloem): Figure out how to represent + // mutation + // contents in DLQ + .addValue( + Iterators.toString( + mutation.getValues().iterator())) + .build()) + .collect(Collectors.toList()))) + .setRowSchema(failureSchema); + return PCollectionRowTuple.of("failures", failures); + } + }; + } + } + + @Override + public @UnknownKeyFor @NonNull @Initialized String identifier() { + return "beam:schematransform:org.apache.beam:spanner_write:v1"; + } + + @Override + public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String> + inputCollectionNames() { + return Collections.singletonList("input"); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String> + outputCollectionNames() { + return Collections.singletonList("failures"); + } + + @AutoValue + @DefaultSchema(AutoValueSchema.class) + public abstract static class SpannerWriteSchemaTransformConfiguration implements Serializable { + public abstract String getInstanceId(); + + public abstract String getDatabaseId(); + + public abstract String getTableId(); + + public static Builder builder() { + return new AutoValue_SpannerWriteSchemaTransformProvider_SpannerWriteSchemaTransformConfiguration + .Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setInstanceId(String instanceId); + + public abstract Builder setDatabaseId(String databaseId); + + public abstract Builder setTableId(String tableId); + + public abstract SpannerWriteSchemaTransformConfiguration build(); + } + } +} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java index f8558ed24402..9594633d2dc4 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java @@ -34,20 +34,26 @@ import com.google.spanner.admin.database.v1.CreateDatabaseMetadata; import java.io.Serializable; import java.util.Collections; +import java.util.Objects; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.View; import org.apache.beam.sdk.transforms.Wait; +import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.PCollectionView; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; @@ -200,6 +206,40 @@ public void testWrite() throws Exception { assertThat(countNumberOfRecords(pgDatabaseName), equalTo((long) numRecords)); } + @Test + public void testWriteViaSchemaTransform() throws Exception { + int numRecords = 100; + final Schema tableSchema = + Schema.builder().addInt64Field("Key").addStringField("Value").build(); + PCollectionRowTuple.of( + "input", + p.apply("Init", GenerateSequence.from(0).to(numRecords)) + .apply( + MapElements.into(TypeDescriptors.rows()) + .via( + seed -> + Row.withSchema(tableSchema) + .addValue(seed) + .addValue(Objects.requireNonNull(seed).toString()) + .build())) + .setRowSchema(tableSchema)) + .apply( + new SpannerWriteSchemaTransformProvider() + .from( + SpannerWriteSchemaTransformProvider.SpannerWriteSchemaTransformConfiguration + .builder() + .setDatabaseId(databaseName) + .setInstanceId(options.getInstanceId()) + .setTableId(options.getTable()) + .build()) + .buildTransform()); + + PipelineResult result = p.run(); + result.waitUntilFinish(); + assertThat(result.getState(), is(PipelineResult.State.DONE)); + assertThat(countNumberOfRecords(databaseName), equalTo((long) numRecords)); + } + @Test public void testSequentialWrite() throws Exception { int numRecords = 100; From f1e12992315f40d194407d1026e61a3da549fb14 Mon Sep 17 00:00:00 2001 From: kn1kn1 Date: Sat, 26 Nov 2022 07:24:27 +0900 Subject: [PATCH 248/269] Update java-multi-language-pipelines.md (#24345) same as https://github.com/apache/beam/pull/24302 Fix mvn command --- .../en/documentation/sdks/java-multi-language-pipelines.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/content/en/documentation/sdks/java-multi-language-pipelines.md b/website/www/site/content/en/documentation/sdks/java-multi-language-pipelines.md index e9cbb9932e4b..e84dcfdb849b 100644 --- a/website/www/site/content/en/documentation/sdks/java-multi-language-pipelines.md +++ b/website/www/site/content/en/documentation/sdks/java-multi-language-pipelines.md @@ -171,7 +171,7 @@ export GCP_REGION= mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.multilanguage.PythonDataframeWordCount \ -Dexec.args="--runner=DataflowRunner --project=$GCP_PROJECT \ - --region=us-central1 \ + --region=$GCP_REGION \ --gcpTempLocation=gs://$GCP_BUCKET/multi-language-beam/tmp \ --output=gs://$GCP_BUCKET/multi-language-beam/output" \ -Pdataflow-runner From 6ec3125be0bd434c418936f51678ce792b6c1674 Mon Sep 17 00:00:00 2001 From: Evgeny Antyshev Date: Mon, 28 Nov 2022 17:22:55 +0300 Subject: [PATCH 249/269] [Tour Of Beam] persistence_key for Pg::SaveSnippet (#24287) * restore CI * gen fix * mock_helper * tests * persistent_key * nit * README * README * review * no secrets --- .../workflows/tour_of_beam_examples_ci.yml | 6 +- learning/tour-of-beam/backend/README.md | 8 +- learning/tour-of-beam/backend/function.go | 4 +- .../integration_tests/auth_emulator.go | 4 +- .../backend/integration_tests/auth_test.go | 99 +++++- .../backend/integration_tests/client_pg.go | 39 ++ .../tour-of-beam/backend/internal/entity.go | 9 +- .../backend/internal/persistence_key.go | 37 ++ .../backend/internal/service/content.go | 24 +- .../service/mock_helper.go} | 20 +- .../backend/internal/service/pg_adapter.go | 3 +- .../backend/internal/storage/datastore.go | 31 +- .../backend/internal/storage/iface.go | 6 +- .../backend/internal/storage/index.yaml | 4 + .../backend/internal/storage/mock.go | 5 +- .../backend/internal/storage/schema.go | 7 +- .../backend/playground_api/api.pb.go | 336 +++++++++--------- 17 files changed, 435 insertions(+), 207 deletions(-) create mode 100644 learning/tour-of-beam/backend/integration_tests/client_pg.go create mode 100644 learning/tour-of-beam/backend/internal/persistence_key.go rename learning/tour-of-beam/backend/{playground_api/helper.go => internal/service/mock_helper.go} (60%) diff --git a/.github/workflows/tour_of_beam_examples_ci.yml b/.github/workflows/tour_of_beam_examples_ci.yml index 43d2e5781eb8..5494eb3cfe4d 100644 --- a/.github/workflows/tour_of_beam_examples_ci.yml +++ b/.github/workflows/tour_of_beam_examples_ci.yml @@ -16,10 +16,10 @@ name: Tour Of Beam Examples CI on: - push: + pull_request: paths: - - ./.github/workflows/playground_examples_ci_reusable.yml - - ./.github/workflows/tour_of_beam_examples_ci.yml + - .github/workflows/playground_examples_ci_reusable.yml + - .github/workflows/tour_of_beam_examples_ci.yml - playground/backend/** - playground/infrastructure/** - learning/tour-of-beam/learning-content/** diff --git a/learning/tour-of-beam/backend/README.md b/learning/tour-of-beam/backend/README.md index 38ff225ad8b2..e6f31d887959 100644 --- a/learning/tour-of-beam/backend/README.md +++ b/learning/tour-of-beam/backend/README.md @@ -86,6 +86,7 @@ Prerequisites: * Billing API * Cloud Functions API * Firebase Admin API + * Secret Manager API - set environment variables: * PROJECT_ID: GCP id * REGION: the region, "us-central1" fe @@ -98,10 +99,11 @@ gcloud datastore indexes create ./internal/storage/index.yaml 2. Deploy cloud functions ``` -for endpoint in getSdkList getContentTree getUnitComplete getUserProgress postUnitComplete postUserCode; do +for endpoint in getSdkList getContentTree getUnitContent getUserProgress postUnitComplete postUserCode; do gcloud functions deploy $endpoint --entry-point $endpoint \ --region $REGION --runtime go116 --allow-unauthenticated \ - --trigger-http --set-env-vars="DATASTORE_PROJECT_ID=$PROJECT_ID,GOOGLE_PROJECT_ID=$PROJECT_ID" + --trigger-http \ + --set-env-vars="DATASTORE_PROJECT_ID=$PROJECT_ID,GOOGLE_PROJECT_ID=$PROJECT_ID" done ``` @@ -110,7 +112,7 @@ done - DATASTORE_PROJECT_ID: Google Cloud PROJECT_ID - GOOGLE_PROJECT_ID: Google Cloud PROJECT_ID (consumed by Firebase Admin SDK) - GOOGLE_APPLICATION_CREDENTIALS: path to json auth key -- TOB_LEARNING_PATH: path the content tree root +- TOB_LEARNING_ROOT: path the content tree root 4. Populate datastore ``` diff --git a/learning/tour-of-beam/backend/function.go b/learning/tour-of-beam/backend/function.go index 2c562ac74b72..2e89cd0c9b05 100644 --- a/learning/tour-of-beam/backend/function.go +++ b/learning/tour-of-beam/backend/function.go @@ -86,7 +86,7 @@ func MakePlaygroundClient(ctx context.Context) pb.PlaygroundServiceClient { // * PLAYGROUND_ROUTER_HOST: playground API host/port if os.Getenv("TOB_MOCK") > "" { fmt.Println("Using mock playground client") - return pb.GetMockClient() + return service.GetMockClient() } else { host := os.Getenv("PLAYGROUND_ROUTER_HOST") cc, err := grpc.Dial(host, grpc.WithTransportCredentials(insecure.NewCredentials())) @@ -227,7 +227,7 @@ func postUserCode(w http.ResponseWriter, r *http.Request, sdk tob.Sdk, uid strin finalizeErrResponse(w, http.StatusNotFound, NOT_FOUND, "unit not found") return } - if err != nil { + if err := errors.Unwrap(err); err != nil { log.Println("Save user code error:", err) message := "storage error" if st, ok := grpc_status.FromError(err); ok { diff --git a/learning/tour-of-beam/backend/integration_tests/auth_emulator.go b/learning/tour-of-beam/backend/integration_tests/auth_emulator.go index 9273153efcd8..0b672a244d5b 100644 --- a/learning/tour-of-beam/backend/integration_tests/auth_emulator.go +++ b/learning/tour-of-beam/backend/integration_tests/auth_emulator.go @@ -113,10 +113,10 @@ func (e *EmulatorClient) do(method, endpoint string, jsonBody map[string]string) // Simulate Frontend client authorization logic // Here, we use the simplest possible authorization: email/password // Firebase Admin SDK lacks methods to create a user and get ID token -func (e *EmulatorClient) getIDToken() string { +func (e *EmulatorClient) getIDToken(email string) string { // create a user (sign-up with dummy email/password) endpoint := "identitytoolkit.googleapis.com/v1/accounts:signUp?key=anything_goes" - body := map[string]string{"email": "a@b.c", "password": "1q2w3e"} + body := map[string]string{"email": email, "password": "1q2w3e"} resp, err := e.do(http.MethodPost, endpoint, body) if err != nil { log.Fatalf("emulator request error: %+v", err) diff --git a/learning/tour-of-beam/backend/integration_tests/auth_test.go b/learning/tour-of-beam/backend/integration_tests/auth_test.go index 6e5c7881d58a..6b0e4df98d29 100644 --- a/learning/tour-of-beam/backend/integration_tests/auth_test.go +++ b/learning/tour-of-beam/backend/integration_tests/auth_test.go @@ -16,11 +16,13 @@ package main import ( + "context" "flag" "net/http" "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -59,7 +61,7 @@ func checkBadHttpCode(t *testing.T, err error, code int) { } func TestSaveGetProgress(t *testing.T) { - idToken := emulator.getIDToken() + idToken := emulator.getIDToken("a@b.c") // postUnitCompleteURL port := os.Getenv(PORT_POST_UNIT_COMPLETE) @@ -134,6 +136,101 @@ func TestSaveGetProgress(t *testing.T) { if err != nil { t.Fatal(err) } + assert.Equal(t, len(exp.Units), len(resp.Units)) + assert.Equal(t, exp.Units[1].Id, resp.Units[1].Id) + // snippet_id is derived from random uid + exp.Units[1].UserSnippetId = resp.Units[1].UserSnippetId assert.Equal(t, exp, resp) }) } + +func TestUserCode(t *testing.T) { + var snippetId1, snippetId2, snippetId3 string + idToken1 := emulator.getIDToken("a1@b.c") + idToken2 := emulator.getIDToken("a2@b.c") + req := makeUserCodeRequest() + originalCode := req.Files[0].Content + + // postUserCodeURL + port := os.Getenv(PORT_POST_USER_CODE) + if port == "" { + t.Fatal(PORT_POST_USER_CODE, "env not set") + } + postUserCodeURL := "http://localhost:" + port + + // getUserProgressURL + port = os.Getenv(PORT_GET_USER_PROGRESS) + if port == "" { + t.Fatal(PORT_GET_USER_PROGRESS, "env not set") + } + getUserProgressURL := "http://localhost:" + port + + t.Run("save_code_user1_example1", func(t *testing.T) { + _, err := PostUserCode(postUserCodeURL, "python", "example1", idToken1, req) + if err != nil { + t.Fatal(err) + } + }) + t.Run("save_code_user2_example1", func(t *testing.T) { + _, err := PostUserCode(postUserCodeURL, "python", "example1", idToken2, req) + if err != nil { + t.Fatal(err) + } + }) + t.Run("check1", func(t *testing.T) { + resp, err := GetUserProgress(getUserProgressURL, "python", idToken1) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "example1", resp.Units[0].Id) + snippetId1 = resp.Units[0].UserSnippetId + }) + t.Run("check2", func(t *testing.T) { + resp, err := GetUserProgress(getUserProgressURL, "python", idToken2) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "example1", resp.Units[0].Id) + snippetId2 = resp.Units[0].UserSnippetId + assert.NotEqual(t, snippetId1, snippetId2, "different users, same snippet ids") + }) + t.Run("save_code_user1_updated", func(t *testing.T) { + // modify snippet code + req.Files[0].Content += "; sys.exit(1)" + + _, err := PostUserCode(postUserCodeURL, "python", "example1", idToken1, req) + if err != nil { + t.Fatal(err) + } + }) + t.Run("check3", func(t *testing.T) { + resp, err := GetUserProgress(getUserProgressURL, "python", idToken1) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "example1", resp.Units[0].Id) + snippetId3 = resp.Units[0].UserSnippetId + assert.NotEqual(t, snippetId1, snippetId3, "updated code, same snippet ids") + }) + t.Run("check_snippet1", func(t *testing.T) { + ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + _, err := GetSnippet(ctx, snippetId1) + assert.NotNil(t, err, "previous snippet available") + + resp, err := GetSnippet(ctx, snippetId3) + assert.Nil(t, err) + assert.Equal(t, req.Files[0].Content, resp.Files[0].Content) + assert.Equal(t, req.Files[0].IsMain, resp.Files[0].IsMain) + assert.Equal(t, req.Files[0].Name, resp.Files[0].Name) + assert.Equal(t, req.PipelineOptions, resp.PipelineOptions) + }) + t.Run("check_snippet2", func(t *testing.T) { + ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + resp, err := GetSnippet(ctx, snippetId2) + assert.Nil(t, err) + assert.Equal(t, originalCode, resp.Files[0].Content) + assert.Equal(t, req.Files[0].IsMain, resp.Files[0].IsMain) + assert.Equal(t, req.Files[0].Name, resp.Files[0].Name) + assert.Equal(t, req.PipelineOptions, resp.PipelineOptions) + }) +} diff --git a/learning/tour-of-beam/backend/integration_tests/client_pg.go b/learning/tour-of-beam/backend/integration_tests/client_pg.go new file mode 100644 index 000000000000..bd22438c2e27 --- /dev/null +++ b/learning/tour-of-beam/backend/integration_tests/client_pg.go @@ -0,0 +1,39 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + + pb "beam.apache.org/learning/tour-of-beam/backend/playground_api" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func GetSnippet(ctx context.Context, snippetId string) (*pb.GetSnippetResponse, error) { + routerHost := os.Getenv("PLAYGROUND_ROUTER_HOST") + conn, err := grpc.DialContext(ctx, routerHost, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("dial grpc: %w", err) + } + client := pb.NewPlaygroundServiceClient(conn) + + req := &pb.GetSnippetRequest{Id: snippetId} + resp, err := client.GetSnippet(ctx, req) + if err != nil { + return nil, fmt.Errorf("get snippet: %w", err) + } + return resp, nil +} diff --git a/learning/tour-of-beam/backend/internal/entity.go b/learning/tour-of-beam/backend/internal/entity.go index 62c33722f930..a6f7bbfb51f5 100644 --- a/learning/tour-of-beam/backend/internal/entity.go +++ b/learning/tour-of-beam/backend/internal/entity.go @@ -15,12 +15,13 @@ package internal -import "errors" +import ( + "errors" +) var ( - ErrNoUnit = errors.New("unit not found") - ErrNoUser = errors.New("user not found") - ErrPlayground = errors.New("playground error") + ErrNoUnit = errors.New("unit not found") + ErrNoUser = errors.New("user not found") ) type SdkItem struct { diff --git a/learning/tour-of-beam/backend/internal/persistence_key.go b/learning/tour-of-beam/backend/internal/persistence_key.go new file mode 100644 index 000000000000..2d2ebf677aaf --- /dev/null +++ b/learning/tour-of-beam/backend/internal/persistence_key.go @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "crypto/rand" + "encoding/base64" +) + +// size in bytes +const PERSISTENCE_KEY_SIZE = 16 + +// Generate random persistence_key, return it as base64 URL-encoded string +// NOT to be exposed on a front-channel! +func GeneratePersistentKey() string { + raw := make([]byte, PERSISTENCE_KEY_SIZE) + _, err := rand.Read(raw) + if err != nil { + panic(err) + } + + // base64 encode to pass as protobuf string + return base64.URLEncoding.EncodeToString(raw) +} diff --git a/learning/tour-of-beam/backend/internal/service/content.go b/learning/tour-of-beam/backend/internal/service/content.go index 3afc106b70ff..4dffa6da7319 100644 --- a/learning/tour-of-beam/backend/internal/service/content.go +++ b/learning/tour-of-beam/backend/internal/service/content.go @@ -82,11 +82,23 @@ func (s *Svc) SaveUserCode(ctx context.Context, sdk tob.Sdk, unitId, uid string, return err } - req := MakePgSaveRequest(userRequest, sdk) - resp, err := s.PgClient.SaveSnippet(ctx, &req) - if err != nil { - return err + // two-phased commit for 2 resources: datastore tb_user_progress entity and Playground::SaveSnippet call + // In a datastore transaction: + // 1. Get tb_user_progress for (sdk, unit, user), if any. + // Otherwise, fill in the default parameters and generate a new persistence_key + // 2. Call Playground::SaveSnippet GRPC call, return a SnippetId + // 3. Upsert into tb_user_progress with the obtained SnippetId and persistence_key + + // callback: Playground::SaveSnippet GRPC call, return snippetID + savePgSnippet := func(persistenceKey string) (string, error) { + req := MakePgSaveRequest(userRequest, sdk, persistenceKey) + resp, err := s.PgClient.SaveSnippet(ctx, &req) + if err != nil { + return "", err + } + fmt.Println("SaveSnippet response:", resp) + return resp.GetId(), nil } - fmt.Println("SaveSnippet response:", resp) - return s.Repo.SaveUserSnippetId(ctx, sdk, unitId, uid, resp.GetId()) + + return s.Repo.SaveUserSnippetId(ctx, sdk, unitId, uid, savePgSnippet) } diff --git a/learning/tour-of-beam/backend/playground_api/helper.go b/learning/tour-of-beam/backend/internal/service/mock_helper.go similarity index 60% rename from learning/tour-of-beam/backend/playground_api/helper.go rename to learning/tour-of-beam/backend/internal/service/mock_helper.go index 896d3ecc5cf0..52b20decf6b6 100644 --- a/learning/tour-of-beam/backend/playground_api/helper.go +++ b/learning/tour-of-beam/backend/internal/service/mock_helper.go @@ -13,26 +13,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -package playground +package service import ( context "context" + pb "beam.apache.org/learning/tour-of-beam/backend/playground_api" grpc "google.golang.org/grpc" ) -func GetMockClient() PlaygroundServiceClient { +// Mock GRPC client which is enabled by TOB_MOCK env flag +func GetMockClient() pb.PlaygroundServiceClient { - return &PlaygroundServiceClientMock{ - SaveSnippetFunc: func(ctx context.Context, in *SaveSnippetRequest, opts ...grpc.CallOption) (*SaveSnippetResponse, error) { - return &SaveSnippetResponse{Id: "snippet_id_1"}, nil + return &pb.PlaygroundServiceClientMock{ + SaveSnippetFunc: func(ctx context.Context, in *pb.SaveSnippetRequest, opts ...grpc.CallOption) (*pb.SaveSnippetResponse, error) { + return &pb.SaveSnippetResponse{Id: "snippet_id_1"}, nil }, - GetSnippetFunc: func(ctx context.Context, in *GetSnippetRequest, opts ...grpc.CallOption) (*GetSnippetResponse, error) { - return &GetSnippetResponse{ - Files: []*SnippetFile{ + GetSnippetFunc: func(ctx context.Context, in *pb.GetSnippetRequest, opts ...grpc.CallOption) (*pb.GetSnippetResponse, error) { + return &pb.GetSnippetResponse{ + Files: []*pb.SnippetFile{ {Name: "main.py", Content: "import sys; sys.exit(0)", IsMain: true}, }, - Sdk: Sdk_SDK_PYTHON, + Sdk: pb.Sdk_SDK_PYTHON, PipelineOptions: "some opts", }, nil }, diff --git a/learning/tour-of-beam/backend/internal/service/pg_adapter.go b/learning/tour-of-beam/backend/internal/service/pg_adapter.go index ca28b260ebd9..2705f66f7127 100644 --- a/learning/tour-of-beam/backend/internal/service/pg_adapter.go +++ b/learning/tour-of-beam/backend/internal/service/pg_adapter.go @@ -22,7 +22,7 @@ import ( pb "beam.apache.org/learning/tour-of-beam/backend/playground_api" ) -func MakePgSaveRequest(userRequest tob.UserCodeRequest, sdk tob.Sdk) pb.SaveSnippetRequest { +func MakePgSaveRequest(userRequest tob.UserCodeRequest, sdk tob.Sdk, persistence_key string) pb.SaveSnippetRequest { filesProto := make([]*pb.SnippetFile, 0) for _, file := range userRequest.Files { filesProto = append(filesProto, @@ -40,5 +40,6 @@ func MakePgSaveRequest(userRequest tob.UserCodeRequest, sdk tob.Sdk) pb.SaveSnip Sdk: pb.Sdk(sdkIdx), Files: filesProto, PipelineOptions: userRequest.PipelineOptions, + PersistenceKey: persistence_key, } } diff --git a/learning/tour-of-beam/backend/internal/storage/datastore.go b/learning/tour-of-beam/backend/internal/storage/datastore.go index 8116580f0e82..6f01332c50f6 100644 --- a/learning/tour-of-beam/backend/internal/storage/datastore.go +++ b/learning/tour-of-beam/backend/internal/storage/datastore.go @@ -321,20 +321,25 @@ func (d *DatastoreDb) GetUserProgress(ctx context.Context, sdk tob.Sdk, uid stri return sdkProgress, nil } -func (d *DatastoreDb) upsertUnitProgress(ctx context.Context, sdk tob.Sdk, unitId, uid string, applyChanges func(*TbUnitProgress)) error { +func (d *DatastoreDb) upsertUnitProgress(ctx context.Context, sdk tob.Sdk, unitId, uid string, + applyChanges func(*TbUnitProgress) error) error { userKey := pgNameKey(TbUserKind, uid, nil) progressKey := datastoreKey(TbUserProgressKind, sdk, unitId, userKey) _, err := d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error { // default entity values progress := TbUnitProgress{ - Sdk: rootSdkKey(sdk), - UnitID: unitId, + Sdk: rootSdkKey(sdk), + UnitID: unitId, + PersistenceKey: tob.GeneratePersistentKey(), } + if err := tx.Get(progressKey, &progress); err != nil && err != datastore.ErrNoSuchEntity { return err } - applyChanges(&progress) + if err := applyChanges(&progress); err != nil { + return err + } if _, err := tx.Put(progressKey, &progress); err != nil { return err } @@ -347,15 +352,25 @@ func (d *DatastoreDb) upsertUnitProgress(ctx context.Context, sdk tob.Sdk, unitI } func (d *DatastoreDb) SetUnitComplete(ctx context.Context, sdk tob.Sdk, unitId, uid string) error { - return d.upsertUnitProgress(ctx, sdk, unitId, uid, func(p *TbUnitProgress) { + return d.upsertUnitProgress(ctx, sdk, unitId, uid, func(p *TbUnitProgress) error { p.IsCompleted = true + return nil }) } -func (d *DatastoreDb) SaveUserSnippetId(ctx context.Context, sdk tob.Sdk, unitId, uid, snippetId string) error { - return d.upsertUnitProgress(ctx, sdk, unitId, uid, func(p *TbUnitProgress) { +func (d *DatastoreDb) SaveUserSnippetId( + ctx context.Context, sdk tob.Sdk, unitId, uid string, externalSave func(string) (string, error), +) error { + applyChanges := func(p *TbUnitProgress) error { + snippetId, err := externalSave(p.PersistenceKey) + if err != nil { + return err + } + p.SnippetId = snippetId - }) + return nil + } + return d.upsertUnitProgress(ctx, sdk, unitId, uid, applyChanges) } // check if the interface is implemented. diff --git a/learning/tour-of-beam/backend/internal/storage/iface.go b/learning/tour-of-beam/backend/internal/storage/iface.go index ceb7230f0f32..1cfc6f631d5c 100644 --- a/learning/tour-of-beam/backend/internal/storage/iface.go +++ b/learning/tour-of-beam/backend/internal/storage/iface.go @@ -32,5 +32,9 @@ type Iface interface { SaveUser(ctx context.Context, uid string) error GetUserProgress(ctx context.Context, sdk tob.Sdk, uid string) (*tob.SdkProgress, error) SetUnitComplete(ctx context.Context, sdk tob.Sdk, unitId, uid string) error - SaveUserSnippetId(ctx context.Context, sdk tob.Sdk, unitId, uid, snippetId string) error + + SaveUserSnippetId( + ctx context.Context, sdk tob.Sdk, unitId, uid string, + externalSave func(string) (string, error), + ) error } diff --git a/learning/tour-of-beam/backend/internal/storage/index.yaml b/learning/tour-of-beam/backend/internal/storage/index.yaml index 109b7b9fb85e..e0f2132745de 100644 --- a/learning/tour-of-beam/backend/internal/storage/index.yaml +++ b/learning/tour-of-beam/backend/internal/storage/index.yaml @@ -11,6 +11,10 @@ indexes: properties: - name: "id" - name: "type" +- kind: "pg_snippets" + properties: + - name: "persistenceKey" + - name: "numberOfFiles" - kind: "tb_learning_module" ancestor: yes properties: diff --git a/learning/tour-of-beam/backend/internal/storage/mock.go b/learning/tour-of-beam/backend/internal/storage/mock.go index 6f0a6e182f5a..d93e1e32e3fd 100644 --- a/learning/tour-of-beam/backend/internal/storage/mock.go +++ b/learning/tour-of-beam/backend/internal/storage/mock.go @@ -83,6 +83,9 @@ func (d *Mock) SetUnitComplete(ctx context.Context, sdk tob.Sdk, unitId, uid str return nil } -func (d *Mock) SaveUserSnippetId(ctx context.Context, sdk tob.Sdk, unitId, uid, snippetId string) error { +func (d *Mock) SaveUserSnippetId( + ctx context.Context, sdk tob.Sdk, unitId, uid string, + externalSave func(string) (string, error), +) error { return nil } diff --git a/learning/tour-of-beam/backend/internal/storage/schema.go b/learning/tour-of-beam/backend/internal/storage/schema.go index e0d05f0ab4f9..7d382e47096e 100644 --- a/learning/tour-of-beam/backend/internal/storage/schema.go +++ b/learning/tour-of-beam/backend/internal/storage/schema.go @@ -109,9 +109,10 @@ type TbUnitProgress struct { Key *datastore.Key `datastore:"__key__"` Sdk *datastore.Key `datastore:"sdk"` - UnitID string `datastore:"unitId"` - IsCompleted bool `datastore:"isCompleted"` - SnippetId string `datastore:"snippetId"` + UnitID string `datastore:"unitId"` + IsCompleted bool `datastore:"isCompleted"` + SnippetId string `datastore:"snippetId"` + PersistenceKey string `datastore:"persistenceKey,noindex"` } type PgSnippets struct { diff --git a/learning/tour-of-beam/backend/playground_api/api.pb.go b/learning/tour-of-beam/backend/playground_api/api.pb.go index f9f402704b5b..c95c91d10930 100644 --- a/learning/tour-of-beam/backend/playground_api/api.pb.go +++ b/learning/tour-of-beam/backend/playground_api/api.pb.go @@ -2191,6 +2191,7 @@ type SaveSnippetRequest struct { // The pipeline options as they would be passed to the program (e.g. "--option1 value1 --option2 value2") PipelineOptions string `protobuf:"bytes,3,opt,name=pipeline_options,json=pipelineOptions,proto3" json:"pipeline_options,omitempty"` Complexity Complexity `protobuf:"varint,4,opt,name=complexity,proto3,enum=api.v1.Complexity" json:"complexity,omitempty"` + PersistenceKey string `protobuf:"bytes,5,opt,name=persistence_key,json=persistenceKey,proto3" json:"persistence_key,omitempty"` } func (x *SaveSnippetRequest) Reset() { @@ -2253,6 +2254,13 @@ func (x *SaveSnippetRequest) GetComplexity() Complexity { return Complexity_COMPLEXITY_UNSPECIFIED } +func (x *SaveSnippetRequest) GetPersistenceKey() string { + if x != nil { + return x.PersistenceKey + } + return "" +} + // SaveSnippetResponse contains information of the generated ID. type SaveSnippetResponse struct { state protoimpl.MessageState @@ -2663,7 +2671,7 @@ var file_api_proto_rawDesc = []byte{ 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4d, 0x61, 0x69, 0x6e, 0x22, 0xbd, 0x01, 0x0a, 0x12, 0x53, + 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4d, 0x61, 0x69, 0x6e, 0x22, 0xe6, 0x01, 0x0a, 0x12, 0x53, 0x61, 0x76, 0x65, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, @@ -2675,176 +2683,178 @@ var file_api_proto_rawDesc = []byte{ 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x69, 0x74, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x69, 0x74, 0x79, 0x52, 0x0a, - 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x69, 0x74, 0x79, 0x22, 0x25, 0x0a, 0x13, 0x53, 0x61, - 0x76, 0x65, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x22, 0x23, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0xbd, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x53, 0x6e, - 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, - 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x46, 0x69, 0x6c, - 0x65, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x03, 0x73, 0x64, 0x6b, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x53, - 0x64, 0x6b, 0x52, 0x03, 0x73, 0x64, 0x6b, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x69, 0x70, 0x65, 0x6c, - 0x69, 0x6e, 0x65, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x69, 0x74, 0x79, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, - 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x69, 0x74, 0x79, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, - 0x6c, 0x65, 0x78, 0x69, 0x74, 0x79, 0x2a, 0x52, 0x0a, 0x03, 0x53, 0x64, 0x6b, 0x12, 0x13, 0x0a, - 0x0f, 0x53, 0x44, 0x4b, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x44, 0x4b, 0x5f, 0x4a, 0x41, 0x56, 0x41, 0x10, 0x01, - 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x44, 0x4b, 0x5f, 0x47, 0x4f, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, - 0x53, 0x44, 0x4b, 0x5f, 0x50, 0x59, 0x54, 0x48, 0x4f, 0x4e, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08, - 0x53, 0x44, 0x4b, 0x5f, 0x53, 0x43, 0x49, 0x4f, 0x10, 0x04, 0x2a, 0xb8, 0x02, 0x0a, 0x06, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, - 0x11, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x41, 0x54, 0x49, - 0x4e, 0x47, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x56, - 0x41, 0x4c, 0x49, 0x44, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, - 0x02, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x52, 0x45, 0x50, - 0x41, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x54, 0x41, 0x54, 0x55, - 0x53, 0x5f, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x45, 0x52, - 0x52, 0x4f, 0x52, 0x10, 0x04, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x43, 0x4f, 0x4d, 0x50, 0x49, 0x4c, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x18, 0x0a, 0x14, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x49, 0x4c, 0x45, 0x5f, 0x45, 0x52, - 0x52, 0x4f, 0x52, 0x10, 0x06, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x45, 0x58, 0x45, 0x43, 0x55, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x12, 0x13, 0x0a, 0x0f, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x46, 0x49, 0x4e, 0x49, 0x53, 0x48, 0x45, 0x44, 0x10, 0x08, - 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x5f, 0x45, - 0x52, 0x52, 0x4f, 0x52, 0x10, 0x09, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, - 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x0a, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x4f, 0x55, 0x54, 0x10, 0x0b, - 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, - 0x4c, 0x45, 0x44, 0x10, 0x0c, 0x2a, 0xae, 0x01, 0x0a, 0x15, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, - 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x27, 0x0a, 0x23, 0x50, 0x52, 0x45, 0x43, 0x4f, 0x4d, 0x50, 0x49, 0x4c, 0x45, 0x44, 0x5f, 0x4f, - 0x42, 0x4a, 0x45, 0x43, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, - 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x23, 0x0a, 0x1f, 0x50, 0x52, 0x45, 0x43, + 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x69, 0x74, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x65, + 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x65, + 0x4b, 0x65, 0x79, 0x22, 0x25, 0x0a, 0x13, 0x53, 0x61, 0x76, 0x65, 0x53, 0x6e, 0x69, 0x70, 0x70, + 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x23, 0x0a, 0x11, 0x47, 0x65, + 0x74, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, + 0xbd, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x53, + 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, + 0x73, 0x12, 0x1d, 0x0a, 0x03, 0x73, 0x64, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x64, 0x6b, 0x52, 0x03, 0x73, 0x64, 0x6b, + 0x12, 0x29, 0x0a, 0x10, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x6f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x0a, 0x63, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x69, 0x74, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, + 0x69, 0x74, 0x79, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x69, 0x74, 0x79, 0x2a, + 0x52, 0x0a, 0x03, 0x53, 0x64, 0x6b, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x44, 0x4b, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, + 0x44, 0x4b, 0x5f, 0x4a, 0x41, 0x56, 0x41, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x44, 0x4b, + 0x5f, 0x47, 0x4f, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x44, 0x4b, 0x5f, 0x50, 0x59, 0x54, + 0x48, 0x4f, 0x4e, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x44, 0x4b, 0x5f, 0x53, 0x43, 0x49, + 0x4f, 0x10, 0x04, 0x2a, 0xb8, 0x02, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, + 0x0a, 0x12, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, + 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x1b, 0x0a, + 0x17, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x41, 0x54, 0x49, + 0x4f, 0x4e, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, + 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, + 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x52, 0x45, 0x50, 0x41, + 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x12, 0x14, + 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x49, 0x4c, 0x49, + 0x4e, 0x47, 0x10, 0x05, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, + 0x4f, 0x4d, 0x50, 0x49, 0x4c, 0x45, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x06, 0x12, 0x14, + 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x58, 0x45, 0x43, 0x55, 0x54, 0x49, + 0x4e, 0x47, 0x10, 0x07, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x46, + 0x49, 0x4e, 0x49, 0x53, 0x48, 0x45, 0x44, 0x10, 0x08, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x41, + 0x54, 0x55, 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x09, 0x12, + 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, + 0x0a, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x5f, + 0x54, 0x49, 0x4d, 0x45, 0x4f, 0x55, 0x54, 0x10, 0x0b, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x54, 0x41, + 0x54, 0x55, 0x53, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x45, 0x44, 0x10, 0x0c, 0x2a, 0xae, + 0x01, 0x0a, 0x15, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x27, 0x0a, 0x23, 0x50, 0x52, 0x45, 0x43, 0x4f, 0x4d, 0x50, 0x49, 0x4c, 0x45, 0x44, 0x5f, 0x4f, 0x42, 0x4a, 0x45, 0x43, 0x54, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x45, 0x58, 0x41, 0x4d, 0x50, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x20, 0x0a, - 0x1c, 0x50, 0x52, 0x45, 0x43, 0x4f, 0x4d, 0x50, 0x49, 0x4c, 0x45, 0x44, 0x5f, 0x4f, 0x42, 0x4a, - 0x45, 0x43, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4b, 0x41, 0x54, 0x41, 0x10, 0x02, 0x12, - 0x25, 0x0a, 0x21, 0x50, 0x52, 0x45, 0x43, 0x4f, 0x4d, 0x50, 0x49, 0x4c, 0x45, 0x44, 0x5f, 0x4f, - 0x42, 0x4a, 0x45, 0x43, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x54, 0x5f, - 0x54, 0x45, 0x53, 0x54, 0x10, 0x03, 0x2a, 0x6e, 0x0a, 0x0a, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, - 0x78, 0x69, 0x74, 0x79, 0x12, 0x1a, 0x0a, 0x16, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x58, 0x49, - 0x54, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x14, 0x0a, 0x10, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x58, 0x49, 0x54, 0x59, 0x5f, 0x42, - 0x41, 0x53, 0x49, 0x43, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, - 0x58, 0x49, 0x54, 0x59, 0x5f, 0x4d, 0x45, 0x44, 0x49, 0x55, 0x4d, 0x10, 0x02, 0x12, 0x17, 0x0a, - 0x13, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x58, 0x49, 0x54, 0x59, 0x5f, 0x41, 0x44, 0x56, 0x41, - 0x4e, 0x43, 0x45, 0x44, 0x10, 0x03, 0x32, 0x8b, 0x0d, 0x0a, 0x11, 0x50, 0x6c, 0x61, 0x79, 0x67, - 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3a, 0x0a, 0x07, - 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, - 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x17, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x64, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x0b, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, - 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x49, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, - 0x12, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, - 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x4f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x47, - 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, - 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x47, 0x72, - 0x61, 0x70, 0x68, 0x12, 0x17, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, - 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x61, + 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x23, 0x0a, 0x1f, 0x50, 0x52, 0x45, 0x43, 0x4f, 0x4d, 0x50, 0x49, 0x4c, 0x45, 0x44, + 0x5f, 0x4f, 0x42, 0x4a, 0x45, 0x43, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x58, 0x41, + 0x4d, 0x50, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x20, 0x0a, 0x1c, 0x50, 0x52, 0x45, 0x43, 0x4f, 0x4d, + 0x50, 0x49, 0x4c, 0x45, 0x44, 0x5f, 0x4f, 0x42, 0x4a, 0x45, 0x43, 0x54, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x4b, 0x41, 0x54, 0x41, 0x10, 0x02, 0x12, 0x25, 0x0a, 0x21, 0x50, 0x52, 0x45, 0x43, + 0x4f, 0x4d, 0x50, 0x49, 0x4c, 0x45, 0x44, 0x5f, 0x4f, 0x42, 0x4a, 0x45, 0x43, 0x54, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x54, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x03, 0x2a, + 0x6e, 0x0a, 0x0a, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x69, 0x74, 0x79, 0x12, 0x1a, 0x0a, + 0x16, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x58, 0x49, 0x54, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x4f, 0x4d, + 0x50, 0x4c, 0x45, 0x58, 0x49, 0x54, 0x59, 0x5f, 0x42, 0x41, 0x53, 0x49, 0x43, 0x10, 0x01, 0x12, + 0x15, 0x0a, 0x11, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x58, 0x49, 0x54, 0x59, 0x5f, 0x4d, 0x45, + 0x44, 0x49, 0x55, 0x4d, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, + 0x58, 0x49, 0x54, 0x59, 0x5f, 0x41, 0x44, 0x56, 0x41, 0x4e, 0x43, 0x45, 0x44, 0x10, 0x03, 0x32, + 0x8b, 0x0d, 0x0a, 0x11, 0x50, 0x6c, 0x61, 0x79, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x64, 0x65, + 0x12, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x46, 0x0a, 0x0b, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x0c, 0x47, 0x65, 0x74, + 0x52, 0x75, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, + 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x12, + 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, + 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x3d, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x17, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x52, 0x75, 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, - 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5e, - 0x0a, 0x13, 0x47, 0x65, 0x74, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, - 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x22, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, - 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, - 0x0a, 0x14, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x75, - 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x4f, - 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, - 0x65, 0x6c, 0x12, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6e, 0x63, - 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x64, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, - 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x24, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, - 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, - 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, + 0x65, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x46, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1a, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x45, 0x72, + 0x72, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5e, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x56, 0x61, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x22, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x56, + 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x50, 0x72, - 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, - 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, - 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, + 0x65, 0x70, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, + 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x70, + 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, + 0x74, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, + 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x10, 0x47, 0x65, + 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x1f, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x69, + 0x6c, 0x65, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x70, + 0x69, 0x6c, 0x65, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x37, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x15, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6e, 0x63, + 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6d, 0x0a, 0x18, 0x47, 0x65, - 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x27, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x28, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, + 0x63, 0x74, 0x73, 0x12, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, + 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, + 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x61, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, + 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, + 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, + 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, + 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x6d, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, + 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, + 0x27, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x64, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x73, 0x0a, 0x1a, 0x47, 0x65, 0x74, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, + 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x73, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, + 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, + 0x12, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, + 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4f, 0x75, + 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, + 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6d, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x50, 0x72, + 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, + 0x6f, 0x67, 0x73, 0x12, 0x27, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, - 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, - 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6d, - 0x0a, 0x18, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, - 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x27, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, - 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, + 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, + 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x70, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, + 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x47, 0x72, + 0x61, 0x70, 0x68, 0x12, 0x28, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x70, 0x0a, - 0x19, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x28, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, - 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, - 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x76, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, 0x65, - 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x2a, - 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, - 0x6c, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, - 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x0b, 0x53, 0x61, 0x76, 0x65, 0x53, - 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x12, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, - 0x53, 0x61, 0x76, 0x65, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x61, 0x76, 0x65, - 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x43, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x12, 0x19, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, - 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x38, 0x5a, 0x36, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x61, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x70, 0x6c, 0x61, 0x79, 0x67, 0x72, 0x6f, 0x75, - 0x6e, 0x64, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x3b, 0x70, 0x6c, 0x61, 0x79, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, + 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, + 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x76, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x44, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, + 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x2a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, + 0x2e, 0x47, 0x65, 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, + 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, + 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, + 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x46, 0x0a, 0x0b, 0x53, 0x61, 0x76, 0x65, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x12, + 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x61, 0x76, 0x65, 0x53, 0x6e, 0x69, + 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x61, 0x76, 0x65, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x53, + 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x12, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x6e, + 0x69, 0x70, 0x70, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x38, 0x5a, + 0x36, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x6f, 0x72, 0x67, + 0x2f, 0x70, 0x6c, 0x61, 0x79, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x2f, 0x62, 0x61, 0x63, 0x6b, + 0x65, 0x6e, 0x64, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x3b, 0x70, 0x6c, 0x61, + 0x79, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( From a388f996ad5172dd9cb6baa9046c58d3928e7062 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 28 Nov 2022 10:15:16 -0500 Subject: [PATCH 250/269] Get postcommits green and unsickbay (#24342) go test --- .../jenkins/job_PostCommit_Python_Examples_Flink.groovy | 1 - sdks/go/test/integration/integration.go | 2 -- sdks/python/apache_beam/examples/complete/tfidf_it_test.py | 1 + .../examples/cookbook/bigquery_side_input_it_test.py | 1 + .../python/apache_beam/examples/cookbook/mergecontacts_test.py | 1 + .../examples/cookbook/multiple_output_pardo_test.py | 1 + sdks/python/test-suites/portable/common.gradle | 3 --- 7 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy index c1a44b8e9d43..779395bf7093 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy @@ -37,7 +37,6 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Flink', gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(":sdks:python:test-suites:portable:flinkExamplesPostCommit") - switches("-PflinkConfDir=$WORKSPACE/src/runners/flink/src/test/resources") commonJobProperties.setGradleSwitches(delegate) } } diff --git a/sdks/go/test/integration/integration.go b/sdks/go/test/integration/integration.go index 26acc5137e60..2253df597bb4 100644 --- a/sdks/go/test/integration/integration.go +++ b/sdks/go/test/integration/integration.go @@ -137,8 +137,6 @@ var portableFilters = []string{ var flinkFilters = []string{ // TODO(https://github.com/apache/beam/issues/20723): Flink tests timing out on reads. "TestXLang_Combine.*", - // TODO(https://github.com/apache/beam/issues/21094): Test fails on post commits: "Insufficient number of network buffers". - "TestXLang_Multi", "TestDebeziumIO_BasicRead", // TODO(BEAM-13215): GCP IOs currently do not work in non-Dataflow portable runners. "TestBigQueryIO.*", diff --git a/sdks/python/apache_beam/examples/complete/tfidf_it_test.py b/sdks/python/apache_beam/examples/complete/tfidf_it_test.py index fe1649bbfa35..3ecbd0c1ecae 100644 --- a/sdks/python/apache_beam/examples/complete/tfidf_it_test.py +++ b/sdks/python/apache_beam/examples/complete/tfidf_it_test.py @@ -42,6 +42,7 @@ class TfIdfIT(unittest.TestCase): @pytest.mark.examples_postcommit + @pytest.mark.sickbay_flink def test_basics(self): test_pipeline = TestPipeline(is_integration_test=True) diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_it_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_it_test.py index 1f58e82e47d9..bce8584db21c 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_it_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_it_test.py @@ -45,6 +45,7 @@ def setUp(self): @pytest.mark.no_xdist @pytest.mark.examples_postcommit + @pytest.mark.sickbay_flink def test_bigquery_side_input_it(self): state_verifier = PipelineStateMatcher(PipelineState.DONE) NUM_GROUPS = 3 diff --git a/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py b/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py index e996084b81cd..a53318a94a85 100644 --- a/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py +++ b/sdks/python/apache_beam/examples/cookbook/mergecontacts_test.py @@ -136,6 +136,7 @@ def normalize_tsv_results(self, tsv_data): return '\n'.join(sorted(lines_out)) + '\n' @pytest.mark.examples_postcommit + @pytest.mark.sickbay_flink def test_mergecontacts(self): test_pipeline = TestPipeline(is_integration_test=True) diff --git a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py index 706cff70ba70..d4f540163a38 100644 --- a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py +++ b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo_test.py @@ -49,6 +49,7 @@ def get_wordcount_results(self, result_path): return results @pytest.mark.examples_postcommit + @pytest.mark.sickbay_flink def test_multiple_output_pardo(self): test_pipeline = TestPipeline(is_integration_test=True) diff --git a/sdks/python/test-suites/portable/common.gradle b/sdks/python/test-suites/portable/common.gradle index 9585d0922046..c87696793f0f 100644 --- a/sdks/python/test-suites/portable/common.gradle +++ b/sdks/python/test-suites/portable/common.gradle @@ -317,9 +317,6 @@ project.tasks.register("postCommitPy${pythonVersionSuffix}IT") { // suppress metric name collision warning logs '\\"org.apache.flink.runtime.metrics.groups\\":\\"ERROR\\"}' ] - if (project.hasProperty('flinkConfDir')) { - pipelineOpts += ["--flink-conf-dir=${project.property('flinkConfDir')}"] - } def cmdArgs = mapToArgString([ "test_opts": testOpts, "suite": "postCommitIT-flink-py${pythonVersionSuffix}", From 858ea050e498d55972faccddf5a2e5ff0e101315 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 28 Nov 2022 10:36:00 -0500 Subject: [PATCH 251/269] Fix workflow cron syntax (#24376) --- .github/workflows/dask_runner_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dask_runner_tests.yml b/.github/workflows/dask_runner_tests.yml index 382437c0c9aa..33d0575e2c8c 100644 --- a/.github/workflows/dask_runner_tests.yml +++ b/.github/workflows/dask_runner_tests.yml @@ -21,7 +21,7 @@ name: Dask Runner Tests on: schedule: - - cron: 'H H * * *' + - cron: '3 7 * * *' pull_request: branches: ['master', 'release-*'] tags: 'v*' From 0a6a2402b5e755463fc02cccb976f88002e0103d Mon Sep 17 00:00:00 2001 From: Evgeny Antyshev Date: Mon, 28 Nov 2022 19:10:23 +0300 Subject: [PATCH 252/269] concurrency (#24332) --- playground/infrastructure/helper.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/playground/infrastructure/helper.py b/playground/infrastructure/helper.py index 53c9a98072e9..90b3679ad841 100644 --- a/playground/infrastructure/helper.py +++ b/playground/infrastructure/helper.py @@ -163,10 +163,18 @@ async def get_statuses(client: GRPCClient, examples: List[Example], concurrency: except (KeyError, ValueError): pass - async with asyncio.Semaphore(concurrency): - for example in examples: - tasks.append(_update_example_status(example, client)) - await tqdm.gather(*tasks) + semaphore = asyncio.Semaphore(concurrency) + + async def _semaphored_task(example): + await semaphore.acquire() + try: + await _update_example_status(example, client) + finally: + semaphore.release() + + for example in examples: + tasks.append(_semaphored_task(example)) + await tqdm.gather(*tasks) def get_tag(filepath) -> Optional[ExampleTag]: From 453fdfe3fadd726b79654d35013db2f04d5f55dd Mon Sep 17 00:00:00 2001 From: Kenneth Knowles Date: Thu, 17 Nov 2022 21:42:39 -0800 Subject: [PATCH 253/269] Check for null in BeamFnDataGrpcMultiplexer --- .../apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexer.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexer.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexer.java index f7e13d2cf987..d71c87a80de2 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexer.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexer.java @@ -17,6 +17,8 @@ */ package org.apache.beam.sdk.fn.data; +import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; + import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -130,7 +132,8 @@ public void close() { private final class InboundObserver implements StreamObserver { @Override public void onNext(BeamFnApi.Elements value) { - for (BeamFnApi.Elements.Data data : value.getDataList()) { + for (BeamFnApi.Elements.Data maybeData : value.getDataList()) { + BeamFnApi.Elements.Data data = checkArgumentNotNull(maybeData); try { LogicalEndpoint key = LogicalEndpoint.data(data.getInstructionId(), data.getTransformId()); From 1df3d11870c50ca3d2fa88f50664179dea6ab650 Mon Sep 17 00:00:00 2001 From: Kenneth Knowles Date: Thu, 17 Nov 2022 21:22:18 -0800 Subject: [PATCH 254/269] Upgrade checkerframework to 3.12.0 A warning suppression key changed, so that is in the same commit. --- .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +- .../complete/datatokenization/transforms/DataProtectors.java | 5 ++++- .../beam/sdk/io/gcp/healthcare/FhirIOPatientEverything.java | 2 +- .../io/gcp/spanner/changestreams/action/ActionFactory.java | 2 +- .../io/gcp/spanner/changestreams/mapper/MapperFactory.java | 2 +- .../changestreams/model/ChangeStreamRecordMetadata.java | 5 ++++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 95fb9f59e3f4..abb32698ddbd 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -461,7 +461,7 @@ class BeamModulePlugin implements Plugin { def aws_java_sdk2_version = "2.17.127" def cassandra_driver_version = "3.10.2" def cdap_version = "6.5.1" - def checkerframework_version = "3.10.0" + def checkerframework_version = "3.12.0" def classgraph_version = "4.8.104" def dbcp2_version = "2.8.0" def errorprone_version = "2.10.0" diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/DataProtectors.java b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/DataProtectors.java index 90efc0575d42..700b94b93dfa 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/DataProtectors.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/DataProtectors.java @@ -144,7 +144,10 @@ public abstract static class Builder { } /** Class implements stateful doFn for data tokenization using remote RPC. */ - @SuppressWarnings("initialization.static.fields.uninitialized") + @SuppressWarnings({ + "initialization.static.fields.uninitialized", + "initialization.static.field.uninitialized" + }) public static class TokenizationFn extends DoFn>, Row> { private static Schema schemaToRpc; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOPatientEverything.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOPatientEverything.java index 8d91536b1b9e..7483224ea4f1 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOPatientEverything.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOPatientEverything.java @@ -183,7 +183,7 @@ static class GetPatientEverythingFn extends DoFn Date: Thu, 17 Nov 2022 19:23:31 -0800 Subject: [PATCH 255/269] Upgrade checkerframework gradle plugin to 0.6.19 --- buildSrc/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index f7992ed6086d..b8df3a0d001b 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -53,7 +53,7 @@ dependencies { runtimeOnly("com.avast.gradle:gradle-docker-compose-plugin:0.14.12") // Enable docker compose tasks runtimeOnly("ca.cutterslade.gradle:gradle-dependency-analyze:1.8.3") // Enable dep analysis runtimeOnly("gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.4.11") // Enable dep vulnerability analysis - runtimeOnly("org.checkerframework:checkerframework-gradle-plugin:0.5.16") // Enable enhanced static checking plugin + runtimeOnly("org.checkerframework:checkerframework-gradle-plugin:0.6.19") // Enable enhanced static checking plugin } // Because buildSrc is built and tested automatically _before_ gradle From 2f72234aaa99a3b7c49ac5caf46d147fc6a42459 Mon Sep 17 00:00:00 2001 From: Andrew Pilloud Date: Tue, 15 Nov 2022 16:48:02 -0800 Subject: [PATCH 256/269] Handle CompleteWorkStatus shutdown signal This fixes crashes caused by ignoring the field. --- .../dataflow/worker/WorkItemStatusClient.java | 25 ++++++++++++++++--- .../worker/WorkItemStatusClientTest.java | 17 +++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClient.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClient.java index b2d2e73bb279..fcf227334cd8 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClient.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClient.java @@ -75,6 +75,7 @@ public class WorkItemStatusClient { private transient String uniqueWorkId = null; private boolean finalStateSent = false; + private boolean wasAskedToAbort = false; private @Nullable BatchModeExecutionContext executionContext; @@ -114,8 +115,12 @@ public synchronized void setWorker( } /** Return the {@link WorkItemServiceState} resulting from sending an error completion status. */ - public synchronized WorkItemServiceState reportError(Throwable e) throws IOException { + public synchronized @Nullable WorkItemServiceState reportError(Throwable e) throws IOException { checkState(!finalStateSent, "cannot reportUpdates after sending a final state"); + if (wasAskedToAbort) { + LOG.info("Service already asked to abort work item, not reporting ignored progress."); + return null; + } WorkItemStatus status = createStatusUpdate(true); // TODO: Provide more structure representation of error, e.g., the serialized exception object. @@ -147,9 +152,13 @@ public synchronized WorkItemServiceState reportError(Throwable e) throws IOExcep } /** Return the {@link WorkItemServiceState} resulting from sending a success completion status. */ - public synchronized WorkItemServiceState reportSuccess() throws IOException { + public synchronized @Nullable WorkItemServiceState reportSuccess() throws IOException { checkState(!finalStateSent, "cannot reportSuccess after sending a final state"); checkState(worker != null, "setWorker should be called before reportSuccess"); + if (wasAskedToAbort) { + LOG.info("Service already asked to abort work item, not reporting ignored progress."); + return null; + } WorkItemStatus status = createStatusUpdate(true); @@ -168,12 +177,16 @@ public synchronized WorkItemServiceState reportSuccess() throws IOException { } /** Return the {@link WorkItemServiceState} resulting from sending a progress update. */ - public synchronized WorkItemServiceState reportUpdate( + public synchronized @Nullable WorkItemServiceState reportUpdate( @Nullable DynamicSplitResult dynamicSplitResult, Duration requestedLeaseDuration) throws Exception { checkState(worker != null, "setWorker should be called before reportUpdate"); checkState(!finalStateSent, "cannot reportUpdates after sending a final state"); checkArgument(requestedLeaseDuration != null, "requestLeaseDuration must be non-null"); + if (wasAskedToAbort) { + LOG.info("Service already asked to abort work item, not reporting ignored progress."); + return null; + } WorkItemStatus status = createStatusUpdate(false); status.setRequestedLeaseDuration(TimeUtil.toCloudDuration(requestedLeaseDuration)); @@ -207,6 +220,12 @@ private static boolean isReadLoopAbortedError(Throwable t) { throws IOException { WorkItemServiceState result = workUnitClient.reportWorkItemStatus(status); if (result != null) { + if (result.getCompleteWorkStatus() != null + && result.getCompleteWorkStatus().getCode() != com.google.rpc.Code.OK.getNumber()) { + LOG.info("Service asked worker to abort with status: {}", result.getCompleteWorkStatus()); + wasAskedToAbort = true; + return result; + } nextReportIndex = result.getNextReportIndex(); if (nextReportIndex == null && !status.getCompleted()) { LOG.error("Missing next work index in {} when reporting {}.", result, status); diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClientTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClientTest.java index 6cc2eae14408..8dbd90285d28 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClientTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClientTest.java @@ -295,6 +295,23 @@ public void reportIndexSequence() throws Exception { assertThat(updates.get(2).getReportIndex(), equalTo(INITIAL_REPORT_INDEX + 8)); } + @Test + public void reportAbort() throws Exception { + when(worker.extractMetricUpdates()).thenReturn(Collections.emptyList()); + statusClient.setWorker(worker, executionContext); + + when(workUnitClient.reportWorkItemStatus(isA(WorkItemStatus.class))) + .thenReturn( + new WorkItemServiceState() + .setCompleteWorkStatus( + new Status() + .setCode(com.google.rpc.Code.ABORTED_VALUE) + .setMessage("Worker was asked to abort!"))); + statusClient.reportUpdate(null, LEASE_DURATION); + + statusClient.reportSuccess(); + } + @Test public void populateMetricUpdatesNoStateSamplerInfo() throws Exception { // When executionContext.getExecutionStateTracker() returns null, we get no metric updates. From d6dc8d08c22f1f2800846f8bdb893dfc539a7da0 Mon Sep 17 00:00:00 2001 From: tvalentyn Date: Mon, 28 Nov 2022 13:16:33 -0800 Subject: [PATCH 257/269] Revert "Force discarding mode in with_fanout without rewindowing." (#24378) --- .../apache_beam/transforms/combiners_test.py | 61 ------------------- sdks/python/apache_beam/transforms/core.py | 14 ++--- 2 files changed, 4 insertions(+), 71 deletions(-) diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py index 2ea0ba862b98..7e0e83542ee9 100644 --- a/sdks/python/apache_beam/transforms/combiners_test.py +++ b/sdks/python/apache_beam/transforms/combiners_test.py @@ -563,67 +563,6 @@ def has_expected_values(actual): assert_that(result, has_expected_values) - def test_combining_with_sliding_windows_and_fanout(self): - options = PipelineOptions() - options.view_as(StandardOptions).streaming = True - with TestPipeline(options=options) as p: - - def has_expected_values(actual): - from hamcrest.core import assert_that as hamcrest_assert - from hamcrest.library.collection import only_contains - ordered = sorted(actual) - - hamcrest_assert( - ordered, - only_contains([0, 1, 2, 3], [0, 1, 2, 3, 5, 6, 7, 8], [5, 6, 7, 8])) - - result = ( - p - | beam.Create([ - window.TimestampedValue(0, Timestamp(seconds=1666707510)), - window.TimestampedValue(1, Timestamp(seconds=1666707511)), - window.TimestampedValue(2, Timestamp(seconds=1666707512)), - window.TimestampedValue(3, Timestamp(seconds=1666707513)), - window.TimestampedValue(5, Timestamp(seconds=1666707515)), - window.TimestampedValue(6, Timestamp(seconds=1666707516)), - window.TimestampedValue(7, Timestamp(seconds=1666707517)), - window.TimestampedValue(8, Timestamp(seconds=1666707518)) - ]) - | beam.WindowInto(window.SlidingWindows(10, 5)) - | beam.CombineGlobally(beam.combiners.ToListCombineFn()). - without_defaults().with_fanout(7)) - assert_that(result, has_expected_values) - - def test_combining_with_session_windows_and_fanout(self): - options = PipelineOptions() - options.view_as(StandardOptions).streaming = True - with TestPipeline(options=options) as p: - - def has_expected_values(actual): - from hamcrest.core import assert_that as hamcrest_assert - from hamcrest.library.collection import only_contains - ordered = sorted(actual) - - hamcrest_assert(ordered, only_contains([0, 1, 2, 3], [5, 6, 7, 8])) - - result = ( - p - | beam.Create([ - window.TimestampedValue(0, Timestamp(seconds=1666707510)), - window.TimestampedValue(1, Timestamp(seconds=1666707511)), - window.TimestampedValue(2, Timestamp(seconds=1666707512)), - window.TimestampedValue(3, Timestamp(seconds=1666707513)), - window.TimestampedValue(5, Timestamp(seconds=1666707515)), - window.TimestampedValue(6, Timestamp(seconds=1666707516)), - window.TimestampedValue(7, Timestamp(seconds=1666707517)), - window.TimestampedValue(8, Timestamp(seconds=1666707518)) - ]) - | beam.WindowInto(window.Sessions(2)) - | beam.CombineGlobally(beam.combiners.ToListCombineFn()). - without_defaults().with_fanout(7)) - - assert_that(result, has_expected_values) - def test_MeanCombineFn_combine(self): with TestPipeline() as p: input = ( diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index 317844170ca9..978e0e1eac39 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -2743,8 +2743,6 @@ def default_label(self): def expand(self, pcoll): from apache_beam.transforms.trigger import AccumulationMode - from apache_beam.transforms.util import _IdentityWindowFn - combine_fn = self._combine_fn fanout_fn = self._fanout_fn @@ -2804,15 +2802,11 @@ def StripNonce(nonce_key_value): precombined_hot = ( hot # Avoid double counting that may happen with stacked accumulating mode. - | 'ForceDiscardingAccumulation' >> WindowInto( - _IdentityWindowFn(pcoll.windowing.windowfn.get_window_coder()), - trigger=pcoll.windowing.triggerfn, - accumulation_mode=AccumulationMode.DISCARDING, - timestamp_combiner=pcoll.windowing.timestamp_combiner, - allowed_lateness=pcoll.windowing.allowed_lateness) + | 'WindowIntoDiscarding' >> WindowInto( + pcoll.windowing, accumulation_mode=AccumulationMode.DISCARDING) | CombinePerKey(PreCombineFn()) - | Map(StripNonce)) - + | Map(StripNonce) + | 'WindowIntoOriginal' >> WindowInto(pcoll.windowing)) return ((cold, precombined_hot) | Flatten() | CombinePerKey(PostCombineFn())) From f7334414b53057e0139b4e70d5bf0c8eaa578e05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 14:27:41 -0800 Subject: [PATCH 258/269] Bump pillow from 9.2.0 to 9.3.0 in /sdks/python/apache_beam/examples/ml-orchestration/kfp/components/preprocessing (#24173) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../kfp/components/preprocessing/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/examples/ml-orchestration/kfp/components/preprocessing/requirements.txt b/sdks/python/apache_beam/examples/ml-orchestration/kfp/components/preprocessing/requirements.txt index 2ebbd8cf2149..76090cc3652e 100644 --- a/sdks/python/apache_beam/examples/ml-orchestration/kfp/components/preprocessing/requirements.txt +++ b/sdks/python/apache_beam/examples/ml-orchestration/kfp/components/preprocessing/requirements.txt @@ -18,4 +18,4 @@ requests==2.28.1 torch==1.12.0 torchvision==0.13.0 numpy==1.22.4 -Pillow==9.2.0 +Pillow==9.3.0 From 6dd5dafed6ce83a5d7f6c97d7e5eb5eb43321cef Mon Sep 17 00:00:00 2001 From: Lukasz Cwik Date: Mon, 28 Nov 2022 16:27:12 -0800 Subject: [PATCH 259/269] Update precombine bencmark to better represent varied workloads (#24343) 1. Represent more data distributions (hot key, uniform, normal, unique) 2. Run longer allowing the JIT to function 3. Have a random ordering of data 4. Use a blackhole to prevent to the JIT from optimizing away the data --- .../harness/jmh/CombinerTableBenchmark.java | 84 ------------- .../jmh/PrecombineGroupingTableBenchmark.java | 112 ++++++++++++++++++ 2 files changed, 112 insertions(+), 84 deletions(-) delete mode 100644 sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/CombinerTableBenchmark.java create mode 100644 sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/PrecombineGroupingTableBenchmark.java diff --git a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/CombinerTableBenchmark.java b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/CombinerTableBenchmark.java deleted file mode 100644 index 13d164c15b34..000000000000 --- a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/CombinerTableBenchmark.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.fn.harness.jmh; - -import static java.util.concurrent.TimeUnit.SECONDS; - -import java.util.ArrayList; -import java.util.List; -import org.apache.beam.fn.harness.Caches; -import org.apache.beam.fn.harness.PrecombineGroupingTable; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.transforms.Combine; -import org.apache.beam.sdk.transforms.Sum; -import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.sdk.values.KV; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; - -public class CombinerTableBenchmark { - @State(Scope.Benchmark) - public static class CombinerTable { - final int numKeys = 1000; - final int numPerKey = 1000; - final Combine.BinaryCombineIntegerFn sumInts = Sum.ofIntegers(); - final PipelineOptions options = PipelineOptionsFactory.create(); - PrecombineGroupingTable groupingTable; - List>> elements; - - @Param({"true", "false"}) - public String globallyWindowed; - - @Setup(Level.Invocation) - public void setUp() { - groupingTable = - PrecombineGroupingTable.combiningAndSampling( - options, - Caches.eternal(), - sumInts, - StringUtf8Coder.of(), - .001, - Boolean.valueOf(globallyWindowed)); - elements = new ArrayList<>(); - for (int i = 0; i < numKeys; i++) { - elements.add(WindowedValue.valueInGlobalWindow(KV.of(Integer.toString(i), i))); - } - } - } - - @Benchmark - @Fork(value = 1) - @Warmup(time = 1, timeUnit = SECONDS) - @Measurement(time = 1, timeUnit = SECONDS) - public void uniformDistribution(CombinerTable table) throws Exception { - for (int i = 0; i < table.numPerKey; i++) { - for (WindowedValue> element : table.elements) { - table.groupingTable.put(element, null); - } - } - } -} diff --git a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/PrecombineGroupingTableBenchmark.java b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/PrecombineGroupingTableBenchmark.java new file mode 100644 index 000000000000..0deaf96f18f8 --- /dev/null +++ b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/PrecombineGroupingTableBenchmark.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.fn.harness.jmh; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import org.apache.beam.fn.harness.Caches; +import org.apache.beam.fn.harness.PrecombineGroupingTable; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.transforms.Combine; +import org.apache.beam.sdk.transforms.Sum; +import org.apache.beam.sdk.util.WindowedValue; +import org.apache.beam.sdk.values.KV; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.infra.Blackhole; + +public class PrecombineGroupingTableBenchmark { + private static final int TOTAL_VALUES = 1_000_000; + + @State(Scope.Benchmark) + public static class SumIntegerBinaryCombine { + final Combine.BinaryCombineIntegerFn sumInts = Sum.ofIntegers(); + final PipelineOptions options = PipelineOptionsFactory.create(); + List>> elements; + + @Param({"true", "false"}) + public String globallyWindowed; + + @Param({"uniform", "normal", "hotKey", "uniqueKeys"}) + public String distribution; + + @Setup(Level.Trial) + public void setUp() { + // Use a stable seed to ensure consistency across benchmark runs + Random random = new Random(-2134890234); + elements = new ArrayList<>(); + switch (distribution) { + case "uniform": + for (int i = 0; i < TOTAL_VALUES; ++i) { + int key = random.nextInt(1000); + elements.add(WindowedValue.valueInGlobalWindow(KV.of(Integer.toString(key), key))); + } + break; + case "normal": + for (int i = 0; i < TOTAL_VALUES; ++i) { + int key = (int) (random.nextGaussian() * 1000); + elements.add(WindowedValue.valueInGlobalWindow(KV.of(Integer.toString(key), key))); + } + break; + case "hotKey": + for (int i = 0; i < TOTAL_VALUES; ++i) { + int key; + if (random.nextBoolean()) { + key = 0; + } else { + key = random.nextInt(1000); + } + elements.add(WindowedValue.valueInGlobalWindow(KV.of(Integer.toString(key), key))); + } + break; + case "uniqueKeys": + for (int i = 0; i < TOTAL_VALUES; ++i) { + elements.add(WindowedValue.valueInGlobalWindow(KV.of(Integer.toString(i), i))); + } + Collections.shuffle(elements, random); + break; + default: + } + } + } + + @Benchmark + public void sumIntegerBinaryCombine(SumIntegerBinaryCombine table, Blackhole blackhole) + throws Exception { + PrecombineGroupingTable groupingTable = + PrecombineGroupingTable.combiningAndSampling( + table.options, + Caches.eternal(), + table.sumInts, + StringUtf8Coder.of(), + .001, + Boolean.valueOf(table.globallyWindowed)); + for (int i = 0, size = table.elements.size(); i < size; ++i) { + groupingTable.put(table.elements.get(i), blackhole::consume); + } + groupingTable.flush(blackhole::consume); + } +} From 6c1e2d4b94ffefb6e13c407e176f8a69a8d60c73 Mon Sep 17 00:00:00 2001 From: reuvenlax Date: Mon, 28 Nov 2022 18:42:49 -0800 Subject: [PATCH 260/269] Merge pull request #24320: update bom to the latest one --- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 12 ++++++------ .../container/license_scripts/dep_urls_java.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index abb32698ddbd..107fbefd164b 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -466,15 +466,15 @@ class BeamModulePlugin implements Plugin { def dbcp2_version = "2.8.0" def errorprone_version = "2.10.0" // Try to keep gax_version consistent with gax-grpc version in google_cloud_platform_libraries_bom - def gax_version = "2.19.2" + def gax_version = "2.19.5" def google_clients_version = "2.0.0" def google_cloud_bigdataoss_version = "2.2.6" // Try to keep google_cloud_spanner_version consistent with google_cloud_spanner_bom in google_cloud_platform_libraries_bom - def google_cloud_spanner_version = "6.31.2" + def google_cloud_spanner_version = "6.33.0" def google_code_gson_version = "2.9.1" def google_oauth_clients_version = "1.34.1" // Try to keep grpc_version consistent with gRPC version in google_cloud_platform_libraries_bom - def grpc_version = "1.49.2" + def grpc_version = "1.50.2" def guava_version = "31.1-jre" def hadoop_version = "2.10.2" def hamcrest_version = "2.1" @@ -490,7 +490,7 @@ class BeamModulePlugin implements Plugin { def postgres_version = "42.2.16" def powermock_version = "2.0.9" // Try to keep protobuf_version consistent with the protobuf version in google_cloud_platform_libraries_bom - def protobuf_version = "3.21.7" + def protobuf_version = "3.21.9" def quickcheck_version = "1.0" def sbe_tool_version = "1.25.1" def singlestore_jdbc_version = "1.1.4" @@ -607,9 +607,9 @@ class BeamModulePlugin implements Plugin { google_cloud_pubsub : "com.google.cloud:google-cloud-pubsub", // google_cloud_platform_libraries_bom sets version google_cloud_pubsublite : "com.google.cloud:google-cloud-pubsublite", // google_cloud_platform_libraries_bom sets version // The GCP Libraries BOM dashboard shows the versions set by the BOM: - // https://storage.googleapis.com/cloud-opensource-java-dashboard/com.google.cloud/libraries-bom/26.1.3/artifact_details.html + // https://storage.googleapis.com/cloud-opensource-java-dashboard/com.google.cloud/libraries-bom/26.1.5/artifact_details.html // Update libraries-bom version on sdks/java/container/license_scripts/dep_urls_java.yaml - google_cloud_platform_libraries_bom : "com.google.cloud:libraries-bom:26.1.3", + google_cloud_platform_libraries_bom : "com.google.cloud:libraries-bom:26.1.5", google_cloud_spanner : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version google_cloud_spanner_test : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version:tests", google_code_gson : "com.google.code.gson:gson:$google_code_gson_version", diff --git a/sdks/java/container/license_scripts/dep_urls_java.yaml b/sdks/java/container/license_scripts/dep_urls_java.yaml index 54ee4333323d..0fcf580320ab 100644 --- a/sdks/java/container/license_scripts/dep_urls_java.yaml +++ b/sdks/java/container/license_scripts/dep_urls_java.yaml @@ -42,7 +42,7 @@ jaxen: '1.1.6': type: "3-Clause BSD" libraries-bom: - '26.1.3': + '26.1.5': license: "https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-opensource-java/master/LICENSE" type: "Apache License 2.0" paranamer: From 2a83576d04bcff60ee1896b0cd010b661fa2522c Mon Sep 17 00:00:00 2001 From: reuvenlax Date: Mon, 28 Nov 2022 20:00:52 -0800 Subject: [PATCH 261/269] Merge pull request #24147: First step in adding schema update to Storage API sink. Refactor code #21395 --- .../sdk/io/gcp/bigquery/AppendClientInfo.java | 68 ++++ .../bigquery/BeamRowToStorageApiProto.java | 144 ++++---- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 12 +- .../sdk/io/gcp/bigquery/BigQueryOptions.java | 6 - .../sdk/io/gcp/bigquery/BigQueryUtils.java | 2 +- .../io/gcp/bigquery/SplittingIterable.java | 31 +- .../StorageApiDynamicDestinations.java | 21 +- .../StorageApiDynamicDestinationsBeamRow.java | 50 ++- ...StorageApiDynamicDestinationsTableRow.java | 201 ++++------- .../sdk/io/gcp/bigquery/StorageApiLoads.java | 12 +- .../gcp/bigquery/StorageApiWritePayload.java | 5 +- .../StorageApiWriteUnshardedRecords.java | 155 ++++---- .../StorageApiWritesShardedRecords.java | 214 ++++++----- .../bigquery/TableRowToStorageApiProto.java | 337 ++++++++++++++---- .../BeamRowToStorageApiProtoTest.java | 28 +- .../io/gcp/bigquery/BigQueryIOWriteTest.java | 273 +++----------- .../io/gcp/bigquery/BigQueryUtilsTest.java | 26 +- .../TableRowToStorageApiProtoTest.java | 82 ++++- 18 files changed, 849 insertions(+), 818 deletions(-) create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java new file mode 100644 index 000000000000..1680ef48e4d2 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigquery; + +import com.google.cloud.bigquery.storage.v1.TableSchema; +import com.google.protobuf.Descriptors; +import java.util.function.Consumer; +import java.util.function.Supplier; +import javax.annotation.Nullable; + +/** + * Container class used by {@link StorageApiWritesShardedRecords} and {@link + * StorageApiWritesShardedRecords} to enapsulate a destination {@link TableSchema} along with a + * {@link BigQueryServices.StreamAppendClient} and other objects needed to write records. + */ +class AppendClientInfo { + @Nullable BigQueryServices.StreamAppendClient streamAppendClient; + @Nullable TableSchema tableSchema; + Consumer closeAppendClient; + Descriptors.Descriptor descriptor; + + public AppendClientInfo( + TableSchema tableSchema, Consumer closeAppendClient) + throws Exception { + this.tableSchema = tableSchema; + this.descriptor = TableRowToStorageApiProto.getDescriptorFromTableSchema(tableSchema, true); + this.closeAppendClient = closeAppendClient; + } + + public AppendClientInfo clearAppendClient() { + if (streamAppendClient != null) { + closeAppendClient.accept(streamAppendClient); + this.streamAppendClient = null; + } + return this; + } + + public AppendClientInfo createAppendClient( + BigQueryServices.DatasetService datasetService, + Supplier getStreamName, + boolean useConnectionPool) + throws Exception { + if (streamAppendClient == null) { + this.streamAppendClient = + datasetService.getStreamAppendClient(getStreamName.get(), descriptor, useConnectionPool); + } + return this; + } + + public void close() { + clearAppendClient(); + } +} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java index 816cbe9d6caf..4ef88e01c760 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java @@ -17,16 +17,11 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; +import com.google.cloud.bigquery.storage.v1.TableFieldSchema; +import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.protobuf.ByteString; -import com.google.protobuf.DescriptorProtos.DescriptorProto; -import com.google.protobuf.DescriptorProtos.FieldDescriptorProto; -import com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Label; -import com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type; -import com.google.protobuf.DescriptorProtos.FileDescriptorProto; import com.google.protobuf.Descriptors.Descriptor; -import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Descriptors.FieldDescriptor; -import com.google.protobuf.Descriptors.FileDescriptor; import com.google.protobuf.DynamicMessage; import java.math.BigDecimal; import java.time.LocalDate; @@ -34,7 +29,6 @@ import java.time.LocalTime; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -52,8 +46,6 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Functions; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Bytes; import org.joda.time.ReadableInstant; @@ -71,38 +63,38 @@ public class BeamRowToStorageApiProto { new BigDecimal("-99999999999999999999999999999.999999999"); // TODO(reuvenlax): Support BIGNUMERIC and GEOGRAPHY types. - static final Map PRIMITIVE_TYPES = - ImmutableMap.builder() - .put(TypeName.INT16, Type.TYPE_INT32) - .put(TypeName.BYTE, Type.TYPE_INT32) - .put(TypeName.INT32, Type.TYPE_INT32) - .put(TypeName.INT64, Type.TYPE_INT64) - .put(TypeName.FLOAT, Type.TYPE_FLOAT) - .put(TypeName.DOUBLE, Type.TYPE_DOUBLE) - .put(TypeName.STRING, Type.TYPE_STRING) - .put(TypeName.BOOLEAN, Type.TYPE_BOOL) - .put(TypeName.DATETIME, Type.TYPE_INT64) - .put(TypeName.BYTES, Type.TYPE_BYTES) - .put(TypeName.DECIMAL, Type.TYPE_BYTES) + static final Map PRIMITIVE_TYPES = + ImmutableMap.builder() + .put(TypeName.INT16, TableFieldSchema.Type.INT64) + .put(TypeName.BYTE, TableFieldSchema.Type.INT64) + .put(TypeName.INT32, TableFieldSchema.Type.INT64) + .put(TypeName.INT64, TableFieldSchema.Type.INT64) + .put(TypeName.FLOAT, TableFieldSchema.Type.DOUBLE) + .put(TypeName.DOUBLE, TableFieldSchema.Type.DOUBLE) + .put(TypeName.STRING, TableFieldSchema.Type.STRING) + .put(TypeName.BOOLEAN, TableFieldSchema.Type.BOOL) + .put(TypeName.DATETIME, TableFieldSchema.Type.DATETIME) + .put(TypeName.BYTES, TableFieldSchema.Type.BYTES) + .put(TypeName.DECIMAL, TableFieldSchema.Type.BIGNUMERIC) .build(); // A map of supported logical types to the protobuf field type. - static final Map LOGICAL_TYPES = - ImmutableMap.builder() - .put(SqlTypes.DATE.getIdentifier(), Type.TYPE_INT32) - .put(SqlTypes.TIME.getIdentifier(), Type.TYPE_INT64) - .put(SqlTypes.DATETIME.getIdentifier(), Type.TYPE_INT64) - .put(SqlTypes.TIMESTAMP.getIdentifier(), Type.TYPE_INT64) - .put(EnumerationType.IDENTIFIER, Type.TYPE_STRING) + static final Map LOGICAL_TYPES = + ImmutableMap.builder() + .put(SqlTypes.DATE.getIdentifier(), TableFieldSchema.Type.DATE) + .put(SqlTypes.TIME.getIdentifier(), TableFieldSchema.Type.TIME) + .put(SqlTypes.DATETIME.getIdentifier(), TableFieldSchema.Type.DATETIME) + .put(SqlTypes.TIMESTAMP.getIdentifier(), TableFieldSchema.Type.TIMESTAMP) + .put(EnumerationType.IDENTIFIER, TableFieldSchema.Type.STRING) .build(); static final Map> PRIMITIVE_ENCODERS = ImmutableMap.>builder() - .put(TypeName.INT16, o -> Integer.valueOf((Short) o)) - .put(TypeName.BYTE, o -> Integer.valueOf((Byte) o)) - .put(TypeName.INT32, Functions.identity()) + .put(TypeName.INT16, o -> ((Short) o).longValue()) + .put(TypeName.BYTE, o -> ((Byte) o).longValue()) + .put(TypeName.INT32, o -> ((Integer) o).longValue()) .put(TypeName.INT64, Functions.identity()) - .put(TypeName.FLOAT, Function.identity()) + .put(TypeName.FLOAT, o -> Double.valueOf(o.toString())) .put(TypeName.DOUBLE, Function.identity()) .put(TypeName.STRING, Function.identity()) .put(TypeName.BOOLEAN, Function.identity()) @@ -134,21 +126,6 @@ public class BeamRowToStorageApiProto { ((EnumerationType) logicalType).toString((EnumerationType.Value) value)) .build(); - /** - * Given a Beam Schema, returns a protocol-buffer Descriptor that can be used to write data using - * the BigQuery Storage API. - */ - public static Descriptor getDescriptorFromSchema(Schema schema) - throws DescriptorValidationException { - DescriptorProto descriptorProto = descriptorSchemaFromBeamSchema(schema); - FileDescriptorProto fileDescriptorProto = - FileDescriptorProto.newBuilder().addMessageType(descriptorProto).build(); - FileDescriptor fileDescriptor = - FileDescriptor.buildFrom(fileDescriptorProto, new FileDescriptor[0]); - - return Iterables.getOnlyElement(fileDescriptor.getMessageTypes()); - } - /** * Given a Beam {@link Row} object, returns a protocol-buffer message that can be used to write * data using the BigQuery Storage streaming API. @@ -159,7 +136,9 @@ public static DynamicMessage messageFromBeamRow(Descriptor descriptor, Row row) for (int i = 0; i < row.getFieldCount(); ++i) { Field beamField = beamSchema.getField(i); FieldDescriptor fieldDescriptor = - Preconditions.checkNotNull(descriptor.findFieldByName(beamField.getName().toLowerCase())); + Preconditions.checkNotNull( + descriptor.findFieldByName(beamField.getName().toLowerCase()), + beamField.getName().toLowerCase()); @Nullable Object value = messageValueFromRowValue(fieldDescriptor, beamField, i, row); if (value != null) { builder.setField(fieldDescriptor, value); @@ -169,27 +148,19 @@ public static DynamicMessage messageFromBeamRow(Descriptor descriptor, Row row) } @VisibleForTesting - static DescriptorProto descriptorSchemaFromBeamSchema(Schema schema) { + static TableSchema protoTableSchemaFromBeamSchema(Schema schema) { Preconditions.checkState(schema.getFieldCount() > 0); - DescriptorProto.Builder descriptorBuilder = DescriptorProto.newBuilder(); - // Create a unique name for the descriptor ('-' characters cannot be used). - descriptorBuilder.setName("D" + UUID.randomUUID().toString().replace("-", "_")); - int i = 1; - List nestedTypes = Lists.newArrayList(); + + TableSchema.Builder builder = TableSchema.newBuilder(); for (Field field : schema.getFields()) { - FieldDescriptorProto.Builder fieldDescriptorProtoBuilder = - fieldDescriptorFromBeamField(field, i++, nestedTypes); - descriptorBuilder.addField(fieldDescriptorProtoBuilder); + builder.addFields(fieldDescriptorFromBeamField(field)); } - nestedTypes.forEach(descriptorBuilder::addNestedType); - return descriptorBuilder.build(); + return builder.build(); } - private static FieldDescriptorProto.Builder fieldDescriptorFromBeamField( - Field field, int fieldNumber, List nestedTypes) { - FieldDescriptorProto.Builder fieldDescriptorBuilder = FieldDescriptorProto.newBuilder(); - fieldDescriptorBuilder = fieldDescriptorBuilder.setName(field.getName().toLowerCase()); - fieldDescriptorBuilder = fieldDescriptorBuilder.setNumber(fieldNumber); + private static TableFieldSchema fieldDescriptorFromBeamField(Field field) { + TableFieldSchema.Builder builder = TableFieldSchema.newBuilder(); + builder = builder.setName(field.getName().toLowerCase()); switch (field.getType().getTypeName()) { case ROW: @@ -197,10 +168,10 @@ private static FieldDescriptorProto.Builder fieldDescriptorFromBeamField( if (rowSchema == null) { throw new RuntimeException("Unexpected null schema!"); } - DescriptorProto nested = descriptorSchemaFromBeamSchema(rowSchema); - nestedTypes.add(nested); - fieldDescriptorBuilder = - fieldDescriptorBuilder.setType(Type.TYPE_MESSAGE).setTypeName(nested.getName()); + builder = builder.setType(TableFieldSchema.Type.STRUCT); + for (Schema.Field nestedField : rowSchema.getFields()) { + builder = builder.addFields(fieldDescriptorFromBeamField(nestedField)); + } break; case ARRAY: case ITERABLE: @@ -211,35 +182,44 @@ private static FieldDescriptorProto.Builder fieldDescriptorFromBeamField( Preconditions.checkState( !Preconditions.checkNotNull(elementType.getTypeName()).isCollectionType(), "Nested arrays not supported by BigQuery."); - return fieldDescriptorFromBeamField( - Field.of(field.getName(), elementType), fieldNumber, nestedTypes) - .setLabel(Label.LABEL_REPEATED); + TableFieldSchema elementFieldSchema = + fieldDescriptorFromBeamField(Field.of(field.getName(), elementType)); + builder = builder.setType(elementFieldSchema.getType()); + builder.addAllFields(elementFieldSchema.getFieldsList()); + builder = builder.setMode(TableFieldSchema.Mode.REPEATED); + break; case LOGICAL_TYPE: @Nullable LogicalType logicalType = field.getType().getLogicalType(); if (logicalType == null) { throw new RuntimeException("Unexpected null logical type " + field.getType()); } - @Nullable Type type = LOGICAL_TYPES.get(logicalType.getIdentifier()); + @Nullable TableFieldSchema.Type type = LOGICAL_TYPES.get(logicalType.getIdentifier()); if (type == null) { throw new RuntimeException("Unsupported logical type " + field.getType()); } - fieldDescriptorBuilder = fieldDescriptorBuilder.setType(type); + builder = builder.setType(type); break; case MAP: throw new RuntimeException("Map types not supported by BigQuery."); default: - @Nullable Type primitiveType = PRIMITIVE_TYPES.get(field.getType().getTypeName()); + @Nullable + TableFieldSchema.Type primitiveType = PRIMITIVE_TYPES.get(field.getType().getTypeName()); if (primitiveType == null) { throw new RuntimeException("Unsupported type " + field.getType()); } - fieldDescriptorBuilder = fieldDescriptorBuilder.setType(primitiveType); + builder = builder.setType(primitiveType); } - if (field.getType().getNullable()) { - fieldDescriptorBuilder = fieldDescriptorBuilder.setLabel(Label.LABEL_OPTIONAL); - } else { - fieldDescriptorBuilder = fieldDescriptorBuilder.setLabel(Label.LABEL_REQUIRED); + if (builder.getMode() != TableFieldSchema.Mode.REPEATED) { + if (field.getType().getNullable()) { + builder = builder.setMode(TableFieldSchema.Mode.NULLABLE); + } else { + builder = builder.setMode(TableFieldSchema.Mode.REQUIRED); + } + } + if (field.getDescription() != null) { + builder = builder.setDescription(field.getDescription()); } - return fieldDescriptorBuilder; + return builder.build(); } @Nullable diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java index e72bc20f780e..fedc898fe973 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java @@ -2668,11 +2668,9 @@ public Write withSuccessfulInsertsPropagation(boolean propagateSuccessful) { } /** - * If true, enables automatically detecting BigQuery table schema updates. If a message with - * unknown fields is processed, the BigQuery table is tabled to see if the schema has been - * updated. This is intended for scenarios in which unknown fields are rare, otherwise calls to - * BigQuery will throttle the pipeline. only supported when using one of the STORAGE_API insert - * methods. + * If true, enables automatically detecting BigQuery table schema updates. Table schema updates + * are usually noticed within several minutes. Only supported when using one of the STORAGE_API + * insert methods. */ public Write withAutoSchemaUpdate(boolean autoSchemaUpdate) { return toBuilder().setAutoSchemaUpdate(autoSchemaUpdate).build(); @@ -3174,9 +3172,7 @@ private WriteResult continueExpandTyped( dynamicDestinations, tableRowWriterFactory.getToRowFn(), getCreateDisposition(), - getIgnoreUnknownValues(), - bqOptions.getSchemaUpdateRetries(), - getAutoSchemaUpdate()); + getIgnoreUnknownValues()); } StorageApiLoads storageApiLoads = diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java index 53cb27136412..f0b3e061597a 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java @@ -133,12 +133,6 @@ public interface BigQueryOptions void setBigQueryProject(String value); - @Description("Specify the number of schema update retries. For internal testing only.") - @Default.Integer(2) - Integer getSchemaUpdateRetries(); - - void setSchemaUpdateRetries(Integer value); - @Description("Maximum (best effort) size of a single append to the storage API.") @Default.Integer(2 * 1024 * 1024) Integer getStorageApiAppendThresholdBytes(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java index 6a340496122b..fbea947d0560 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java @@ -576,7 +576,7 @@ public static TableRow toTableRow(Row row) { case DOUBLE: // The above types have native representations in JSON for all their // possible values. - return fieldValue; + return fieldValue.toString(); case STRING: case INT64: diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SplittingIterable.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SplittingIterable.java index 1166647f6238..03b009797b54 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SplittingIterable.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SplittingIterable.java @@ -19,12 +19,8 @@ import com.google.cloud.bigquery.storage.v1.ProtoRows; import com.google.protobuf.ByteString; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; import java.util.Iterator; import java.util.NoSuchElementException; -import java.util.function.Function; -import org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations.DescriptorWrapper; /** * Takes in an iterable and batches the results into multiple ProtoRows objects. The splitSize @@ -34,18 +30,10 @@ class SplittingIterable implements Iterable { private final Iterable underlying; private final long splitSize; - private final Function updateSchema; - private DescriptorWrapper currentDescriptor; - public SplittingIterable( - Iterable underlying, - long splitSize, - DescriptorWrapper currentDescriptor, - Function updateSchema) { + public SplittingIterable(Iterable underlying, long splitSize) { this.underlying = underlying; this.splitSize = splitSize; - this.updateSchema = updateSchema; - this.currentDescriptor = currentDescriptor; } @Override @@ -68,23 +56,8 @@ public ProtoRows next() { long bytesSize = 0; while (underlyingIterator.hasNext()) { StorageApiWritePayload payload = underlyingIterator.next(); - if (payload.getSchemaHash() != currentDescriptor.hash) { - // Schema doesn't match. Try and get an updated schema hash (from the base table). - currentDescriptor = updateSchema.apply(payload.getSchemaHash()); - // Validate that the record can now be parsed. - try { - DynamicMessage msg = - DynamicMessage.parseFrom(currentDescriptor.descriptor, payload.getPayload()); - if (msg.getUnknownFields() != null && !msg.getUnknownFields().asMap().isEmpty()) { - throw new RuntimeException( - "Record schema does not match table. Unknown fields: " - + msg.getUnknownFields()); - } - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } ByteString byteString = ByteString.copyFrom(payload.getPayload()); + inserts.addSerializedRows(byteString); bytesSize += byteString.size(); if (bytesSize > splitSize) { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinations.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinations.java index 53d640a00f27..c3076e8af863 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinations.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinations.java @@ -19,7 +19,6 @@ import com.google.api.services.bigquery.model.TableRow; import com.google.api.services.bigquery.model.TableSchema; -import com.google.protobuf.Descriptors.Descriptor; import java.util.List; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; @@ -31,26 +30,8 @@ /** Base dynamicDestinations class used by the Storage API sink. */ abstract class StorageApiDynamicDestinations extends DynamicDestinations { - /** Container object that contains a proto descriptor along with its deterministic hash. */ - public static class DescriptorWrapper { - public final Descriptor descriptor; - public final long hash; - - public DescriptorWrapper(Descriptor descriptor, long hash) { - this.descriptor = descriptor; - this.hash = hash; - } - - @Override - public String toString() { - return "Descriptor: " + descriptor.getFullName() + " hash: " + hash; - } - } - public interface MessageConverter { - DescriptorWrapper getSchemaDescriptor(); - - void refreshSchema(long expectedHash) throws Exception; + com.google.cloud.bigquery.storage.v1.TableSchema getTableSchema(); StorageApiWritePayload toMessage(T element) throws Exception; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsBeamRow.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsBeamRow.java index 5f85cc1eb1b1..4280d356bd2a 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsBeamRow.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsBeamRow.java @@ -18,6 +18,7 @@ package org.apache.beam.sdk.io.gcp.bigquery; import com.google.api.services.bigquery.model.TableRow; +import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Message; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; @@ -29,7 +30,7 @@ /** Storage API DynamicDestinations used when the input is a Beam Row. */ class StorageApiDynamicDestinationsBeamRow extends StorageApiDynamicDestinations { - private final Schema schema; + private final TableSchema tableSchema; private final SerializableFunction toRow; StorageApiDynamicDestinationsBeamRow( @@ -37,40 +38,37 @@ class StorageApiDynamicDestinationsBeamRow toRow) { super(inner); - this.schema = schema; + this.tableSchema = BeamRowToStorageApiProto.protoTableSchemaFromBeamSchema(schema); this.toRow = toRow; } @Override public MessageConverter getMessageConverter( DestinationT destination, DatasetService datasetService) throws Exception { - return new MessageConverter() { - final Descriptor descriptor; - final long descriptorHash; + return new BeamRowConverter(); + } - { - descriptor = BeamRowToStorageApiProto.getDescriptorFromSchema(schema); - descriptorHash = BigQueryUtils.hashSchemaDescriptorDeterministic(descriptor); - } + class BeamRowConverter implements MessageConverter { + final Descriptor descriptor; - @Override - public DescriptorWrapper getSchemaDescriptor() { - return new DescriptorWrapper(descriptor, descriptorHash); - } + BeamRowConverter() throws Exception { + this.descriptor = TableRowToStorageApiProto.getDescriptorFromTableSchema(tableSchema, true); + } - @Override - public void refreshSchema(long expectedHash) {} + @Override + public TableSchema getTableSchema() { + return tableSchema; + } - @Override - public StorageApiWritePayload toMessage(T element) { - Message msg = BeamRowToStorageApiProto.messageFromBeamRow(descriptor, toRow.apply(element)); - return new AutoValue_StorageApiWritePayload(msg.toByteArray(), descriptorHash); - } + @Override + public StorageApiWritePayload toMessage(T element) { + Message msg = BeamRowToStorageApiProto.messageFromBeamRow(descriptor, toRow.apply(element)); + return new AutoValue_StorageApiWritePayload(msg.toByteArray()); + } - @Override - public TableRow toTableRow(T element) { - return BigQueryUtils.toTableRow(toRow.apply(element)); - } - }; - } + @Override + public TableRow toTableRow(T element) { + return BigQueryUtils.toTableRow(toRow.apply(element)); + } + }; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsTableRow.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsTableRow.java index b025d01f02b3..6797bd20e682 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsTableRow.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsTableRow.java @@ -25,26 +25,20 @@ import java.util.concurrent.ExecutionException; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; -import org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaTooNarrowException; import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class StorageApiDynamicDestinationsTableRow extends StorageApiDynamicDestinations { private final SerializableFunction formatFunction; private final CreateDisposition createDisposition; private final boolean ignoreUnknownValues; - private final int schemaUpdateRetries; - private final boolean autoSchemaUpdates; private static final TableSchemaCache SCHEMA_CACHE = new TableSchemaCache(Duration.standardSeconds(1)); - private static final Logger LOG = - LoggerFactory.getLogger(StorageApiDynamicDestinationsTableRow.class); static { SCHEMA_CACHE.start(); @@ -54,15 +48,11 @@ public class StorageApiDynamicDestinationsTableRow inner, SerializableFunction formatFunction, CreateDisposition createDisposition, - boolean ignoreUnknownValues, - int schemaUpdateRetries, - boolean autoSchemaUpdates) { + boolean ignoreUnknownValues) { super(inner); this.formatFunction = formatFunction; this.createDisposition = createDisposition; this.ignoreUnknownValues = ignoreUnknownValues; - this.schemaUpdateRetries = schemaUpdateRetries; - this.autoSchemaUpdates = autoSchemaUpdates; } static void clearSchemaCache() throws ExecutionException, InterruptedException { @@ -72,128 +62,85 @@ static void clearSchemaCache() throws ExecutionException, InterruptedException { @Override public MessageConverter getMessageConverter( DestinationT destination, DatasetService datasetService) throws Exception { - return new MessageConverter() { - @Nullable TableSchema tableSchema; - TableRowToStorageApiProto.SchemaInformation schemaInformation; - Descriptor descriptor; - long descriptorHash; + return new TableRowConverter(destination, datasetService); + } - { - tableSchema = getSchema(destination); - TableReference tableReference = getTable(destination).getTableReference(); - if (tableSchema == null) { - // If the table already exists, then try and fetch the schema from the existing - // table. - tableSchema = SCHEMA_CACHE.getSchema(tableReference, datasetService); - if (tableSchema == null) { - if (createDisposition == CreateDisposition.CREATE_NEVER) { - throw new RuntimeException( - "BigQuery table " - + tableReference - + " not found. If you wanted to " - + "automatically create the table, set the create disposition to CREATE_IF_NEEDED and specify a " - + "schema."); - } else { - throw new RuntimeException( - "Schema must be set for table " - + tableReference - + " when writing TableRows using Storage API and " - + "using a create disposition of CREATE_IF_NEEDED."); - } - } - } else { - // Make sure we register this schema with the cache, unless there's already a more - // up-to-date schema. - tableSchema = - MoreObjects.firstNonNull( - SCHEMA_CACHE.putSchemaIfAbsent(tableReference, tableSchema), tableSchema); - } - schemaInformation = - TableRowToStorageApiProto.SchemaInformation.fromTableSchema(tableSchema); - descriptor = TableRowToStorageApiProto.getDescriptorFromTableSchema(tableSchema); - descriptorHash = BigQueryUtils.hashSchemaDescriptorDeterministic(descriptor); - } + class TableRowConverter implements MessageConverter { + final @Nullable TableSchema tableSchema; + final com.google.cloud.bigquery.storage.v1.TableSchema protoTableSchema; + final TableRowToStorageApiProto.SchemaInformation schemaInformation; + final Descriptor descriptor; - @Override - public DescriptorWrapper getSchemaDescriptor() { - synchronized (this) { - return new DescriptorWrapper(descriptor, descriptorHash); - } - } + TableRowConverter( + TableSchema tableSchema, + TableRowToStorageApiProto.SchemaInformation schemaInformation, + Descriptor descriptor) { + this.tableSchema = tableSchema; + this.protoTableSchema = TableRowToStorageApiProto.schemaToProtoTableSchema(tableSchema); + this.schemaInformation = schemaInformation; + this.descriptor = descriptor; + } - @Override - public void refreshSchema(long expectedHash) throws Exception { - // When a table is updated, all streams writing to that table will try to refresh the - // schema. Since we don't want them all querying the table for the schema, keep track of - // the expected hash and return if it already matches. - synchronized (this) { - if (expectedHash == descriptorHash) { - return; + TableRowConverter(DestinationT destination, DatasetService datasetService) throws Exception { + TableSchema localTableSchema = getSchema(destination); + TableReference tableReference = getTable(destination).getTableReference(); + if (localTableSchema == null) { + // If the table already exists, then try and fetch the schema from the existing + // table. + localTableSchema = SCHEMA_CACHE.getSchema(tableReference, datasetService); + if (localTableSchema == null) { + if (createDisposition == CreateDisposition.CREATE_NEVER) { + throw new RuntimeException( + "BigQuery table " + + tableReference + + " not found. If you wanted to " + + "automatically create the table, set the create disposition to CREATE_IF_NEEDED and specify a " + + "schema."); + } else { + throw new RuntimeException( + "Schema must be set for table " + + tableReference + + " when writing TableRows using Storage API and " + + "using a create disposition of CREATE_IF_NEEDED."); } } - refreshSchemaInternal(); + } else { + // Make sure we register this schema with the cache, unless there's already a more + // up-to-date schema. + localTableSchema = + MoreObjects.firstNonNull( + SCHEMA_CACHE.putSchemaIfAbsent(tableReference, localTableSchema), localTableSchema); } + this.tableSchema = localTableSchema; + this.protoTableSchema = TableRowToStorageApiProto.schemaToProtoTableSchema(tableSchema); + schemaInformation = + TableRowToStorageApiProto.SchemaInformation.fromTableSchema(protoTableSchema); + descriptor = + TableRowToStorageApiProto.getDescriptorFromTableSchema( + Preconditions.checkStateNotNull(tableSchema), true); + } - public void refreshSchemaInternal() throws Exception { - TableReference tableReference = getTable(destination).getTableReference(); - SCHEMA_CACHE.refreshSchema(tableReference, datasetService); - TableSchema newSchema = SCHEMA_CACHE.getSchema(tableReference, datasetService); - if (newSchema == null) { - throw new RuntimeException("BigQuery table " + tableReference + " not found"); - } - synchronized (this) { - tableSchema = newSchema; - schemaInformation = - TableRowToStorageApiProto.SchemaInformation.fromTableSchema(tableSchema); - descriptor = TableRowToStorageApiProto.getDescriptorFromTableSchema(tableSchema); - long newHash = BigQueryUtils.hashSchemaDescriptorDeterministic(descriptor); - if (descriptorHash != newHash) { - LOG.info( - "Refreshed table " - + BigQueryHelpers.toTableSpec(tableReference) - + " has a new schema."); - } - descriptorHash = newHash; - } - } + @Override + public com.google.cloud.bigquery.storage.v1.TableSchema getTableSchema() { + return protoTableSchema; + } - @Override - public TableRow toTableRow(T element) { - return formatFunction.apply(element); - } + @Override + public TableRow toTableRow(T element) { + return formatFunction.apply(element); + } - @Override - public StorageApiWritePayload toMessage(T element) throws Exception { - int attempt = 0; - do { - TableRowToStorageApiProto.SchemaInformation localSchemaInformation; - Descriptor localDescriptor; - long localDescriptorHash; - synchronized (this) { - localSchemaInformation = schemaInformation; - localDescriptor = descriptor; - localDescriptorHash = descriptorHash; - } - try { - Message msg = - TableRowToStorageApiProto.messageFromTableRow( - localSchemaInformation, - localDescriptor, - formatFunction.apply(element), - ignoreUnknownValues); - return new AutoValue_StorageApiWritePayload(msg.toByteArray(), localDescriptorHash); - } catch (SchemaTooNarrowException e) { - if (!autoSchemaUpdates || attempt > schemaUpdateRetries) { - throw e; - } - // The input record has fields not found in the schema, and ignoreUnknownValues=false. - // It's possible that the user has updated the target table with a wider schema. Try - // to read the target's table schema to see if that is the case. - refreshSchemaInternal(); - ++attempt; - } - } while (true); - } - }; - } + @Override + public StorageApiWritePayload toMessage(T element) throws Exception { + return toMessage(formatFunction.apply(element), true); + } + + public StorageApiWritePayload toMessage(TableRow tableRow, boolean respectRequired) + throws Exception { + Message msg = + TableRowToStorageApiProto.messageFromTableRow( + schemaInformation, descriptor, tableRow, ignoreUnknownValues); + return StorageApiWritePayload.of(msg.toByteArray()); + } + }; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java index 20ab251c9c0c..da2f695f7087 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java @@ -41,7 +41,6 @@ /** This {@link PTransform} manages loads into BigQuery using the Storage API. */ public class StorageApiLoads extends PTransform>, WriteResult> { - static final int MAX_BATCH_SIZE_BYTES = 2 * 1024 * 1024; final TupleTag> successfulRowsTag = new TupleTag<>("successfulRows"); final TupleTag failedRowsTag = new TupleTag<>("failedRows"); @@ -162,6 +161,12 @@ public WriteResult expandTriggered( PCollection, Iterable>> groupedRecords; + int maxAppendBytes = + input + .getPipeline() + .getOptions() + .as(BigQueryOptions.class) + .getStorageApiAppendThresholdBytes(); if (this.allowAutosharding) { groupedRecords = convertMessagesResult @@ -169,7 +174,7 @@ public WriteResult expandTriggered( .apply( "GroupIntoBatches", GroupIntoBatches.ofByteSize( - MAX_BATCH_SIZE_BYTES, + maxAppendBytes, (StorageApiWritePayload e) -> (long) e.getPayload().length) .withMaxBufferingDuration(triggeringFrequency) .withShardedKey()); @@ -182,8 +187,7 @@ public WriteResult expandTriggered( shardedRecords.apply( "GroupIntoBatches", GroupIntoBatches., StorageApiWritePayload>ofByteSize( - MAX_BATCH_SIZE_BYTES, - (StorageApiWritePayload e) -> (long) e.getPayload().length) + maxAppendBytes, (StorageApiWritePayload e) -> (long) e.getPayload().length) .withMaxBufferingDuration(triggeringFrequency)); } PCollectionTuple writeRecordsResult = diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritePayload.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritePayload.java index 4b620a5c6d46..00a34b9c14f7 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritePayload.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritePayload.java @@ -18,6 +18,7 @@ package org.apache.beam.sdk.io.gcp.bigquery; import com.google.auto.value.AutoValue; +import java.io.IOException; import org.apache.beam.sdk.schemas.AutoValueSchema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; @@ -28,5 +29,7 @@ public abstract class StorageApiWritePayload { @SuppressWarnings("mutable") public abstract byte[] getPayload(); - public abstract long getSchemaHash(); + static StorageApiWritePayload of(byte[] payload) throws IOException { + return new AutoValue_StorageApiWritePayload(payload); + } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java index 0f86b8871f0e..99317a3fb23e 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java @@ -25,6 +25,7 @@ import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; import com.google.cloud.bigquery.storage.v1.Exceptions; import com.google.cloud.bigquery.storage.v1.ProtoRows; +import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.cloud.bigquery.storage.v1.WriteStream.Type; import com.google.protobuf.ByteString; import com.google.protobuf.DynamicMessage; @@ -46,7 +47,6 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.StreamAppendClient; import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.RetryType; -import org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations.DescriptorWrapper; import org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations.MessageConverter; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Distribution; @@ -99,18 +99,17 @@ public class StorageApiWriteUnshardedRecords * StreamAppendClient after looking up the cache, and we must ensure that the cache is not * accessed in between the lookup and the pin (any access of the cache could trigger element * expiration). Therefore most used of APPEND_CLIENTS should synchronize. - * - *

TODO(reuvenlax); Once all uses of StreamWriter are using */ - private static final Cache APPEND_CLIENTS = + private static final Cache APPEND_CLIENTS = CacheBuilder.newBuilder() .expireAfterAccess(15, TimeUnit.MINUTES) .removalListener( - (RemovalNotification removal) -> { + (RemovalNotification removal) -> { LOG.info("Expiring append client for " + removal.getKey()); - final @Nullable StreamAppendClient streamAppendClient = removal.getValue(); - // Close the writer in a different thread so as not to block the main one. - runAsyncIgnoreFailure(closeWriterExecutor, streamAppendClient::close); + final @Nullable AppendClientInfo appendClientInfo = removal.getValue(); + if (appendClientInfo != null) { + appendClientInfo.close(); + } }) .build(); @@ -199,9 +198,8 @@ public AppendRowsContext(long offset, ProtoRows protoRows) { class DestinationState { private final String tableUrn; - private final MessageConverter messageConverter; private String streamName = ""; - private @Nullable StreamAppendClient streamAppendClient = null; + private @Nullable AppendClientInfo appendClientInfo = null; private long currentOffset = 0; private List pendingMessages; private transient @Nullable DatasetService maybeDatasetService; @@ -209,8 +207,6 @@ class DestinationState { Metrics.counter(WriteRecordsDoFn.class, "recordsAppended"); private final Counter appendFailures = Metrics.counter(WriteRecordsDoFn.class, "appendFailures"); - private final Counter schemaMismatches = - Metrics.counter(WriteRecordsDoFn.class, "schemaMismatches"); private final Distribution inflightWaitSecondsDistribution = Metrics.distribution(WriteRecordsDoFn.class, "streamWriterWaitSeconds"); private final Counter rowsSentToFailedRowsCollection = @@ -219,7 +215,7 @@ class DestinationState { "rowsSentToFailedRowsCollection"); private final boolean useDefaultStream; - private DescriptorWrapper descriptorWrapper; + private TableSchema tableSchema; private Instant nextCacheTickle = Instant.MAX; private final int clientNumber; private final boolean usingMultiplexing; @@ -232,13 +228,13 @@ public DestinationState( boolean useDefaultStream, int streamAppendClientCount, boolean usingMultiplexing, - long maxRequestSize) { + long maxRequestSize) + throws Exception { this.tableUrn = tableUrn; - this.messageConverter = messageConverter; this.pendingMessages = Lists.newArrayList(); this.maybeDatasetService = datasetService; this.useDefaultStream = useDefaultStream; - this.descriptorWrapper = messageConverter.getSchemaDescriptor(); + this.tableSchema = messageConverter.getTableSchema(); this.clientNumber = new Random().nextInt(streamAppendClientCount); this.usingMultiplexing = usingMultiplexing; this.maxRequestSize = maxRequestSize; @@ -246,9 +242,11 @@ public DestinationState( void teardown() { maybeTickleCache(); - if (streamAppendClient != null) { - runAsyncIgnoreFailure(closeWriterExecutor, streamAppendClient::unpin); - streamAppendClient = null; + if (appendClientInfo != null) { + if (appendClientInfo.streamAppendClient != null) { + runAsyncIgnoreFailure(closeWriterExecutor, appendClientInfo.streamAppendClient::unpin); + } + appendClientInfo = null; } } @@ -279,41 +277,52 @@ String getOrCreateStreamName() { return this.streamName; } - StreamAppendClient generateClient() throws Exception { + AppendClientInfo generateClient(boolean createAppendClient) throws Exception { Preconditions.checkStateNotNull(maybeDatasetService); - return maybeDatasetService.getStreamAppendClient( - streamName, descriptorWrapper.descriptor, usingMultiplexing); + AppendClientInfo appendClientInfo = + new AppendClientInfo( + tableSchema, + // Make sure that the client is always closed in a different thread to avoid + // blocking. + client -> runAsyncIgnoreFailure(closeWriterExecutor, client::close)); + if (createAppendClient) { + appendClientInfo = + appendClientInfo.createAppendClient( + maybeDatasetService, () -> streamName, usingMultiplexing); + Preconditions.checkStateNotNull(appendClientInfo.streamAppendClient).pin(); + } + return appendClientInfo; } - StreamAppendClient getStreamAppendClient(boolean lookupCache) { + AppendClientInfo getAppendClientInfo(boolean lookupCache, boolean createAppendClient) { try { - if (this.streamAppendClient == null) { + if (this.appendClientInfo == null) { getOrCreateStreamName(); - final StreamAppendClient newStreamAppendClient; + final AppendClientInfo newAppendClientInfo; synchronized (APPEND_CLIENTS) { if (lookupCache) { - newStreamAppendClient = + newAppendClientInfo = APPEND_CLIENTS.get( - getStreamAppendClientCacheEntryKey(), () -> generateClient()); + getStreamAppendClientCacheEntryKey(), + () -> generateClient(createAppendClient)); } else { - newStreamAppendClient = generateClient(); - // override the clients in the cache - APPEND_CLIENTS.put(getStreamAppendClientCacheEntryKey(), newStreamAppendClient); + newAppendClientInfo = generateClient(createAppendClient); + // override the clients in the cache. + APPEND_CLIENTS.put(getStreamAppendClientCacheEntryKey(), newAppendClientInfo); } - newStreamAppendClient.pin(); } this.currentOffset = 0; nextCacheTickle = Instant.now().plus(java.time.Duration.ofMinutes(1)); - this.streamAppendClient = newStreamAppendClient; + this.appendClientInfo = newAppendClientInfo; } - return streamAppendClient; + return appendClientInfo; } catch (Exception e) { throw new RuntimeException(e); } } void maybeTickleCache() { - if (streamAppendClient != null && Instant.now().isAfter(nextCacheTickle)) { + if (appendClientInfo != null && Instant.now().isAfter(nextCacheTickle)) { synchronized (APPEND_CLIENTS) { APPEND_CLIENTS.getIfPresent(getStreamAppendClientCacheEntryKey()); } @@ -322,57 +331,36 @@ void maybeTickleCache() { } void invalidateWriteStream() { - if (streamAppendClient != null) { + if (appendClientInfo != null) { synchronized (APPEND_CLIENTS) { // Unpin in a different thread, as it may execute a blocking close. - runAsyncIgnoreFailure(closeWriterExecutor, streamAppendClient::unpin); + if (appendClientInfo.streamAppendClient != null) { + runAsyncIgnoreFailure( + closeWriterExecutor, appendClientInfo.streamAppendClient::unpin); + } // The default stream is cached across multiple different DoFns. If they all try and - // invalidate, then we can - // get races between threads invalidating and recreating streams. For this reason, we - // check to see that the - // cache still contains the object we created before invalidating (in case another - // thread has already invalidated - // and recreated the stream). + // invalidate, then we can get races between threads invalidating and recreating + // streams. For this reason, + // we check to see that the cache still contains the object we created before + // invalidating (in case another + // thread has already invalidated and recreated the stream). String cacheEntryKey = getStreamAppendClientCacheEntryKey(); @Nullable - StreamAppendClient cachedAppendClient = APPEND_CLIENTS.getIfPresent(cacheEntryKey); + AppendClientInfo cachedAppendClient = APPEND_CLIENTS.getIfPresent(cacheEntryKey); if (cachedAppendClient != null && System.identityHashCode(cachedAppendClient) - == System.identityHashCode(streamAppendClient)) { + == System.identityHashCode(appendClientInfo)) { APPEND_CLIENTS.invalidate(cacheEntryKey); } } - streamAppendClient = null; + appendClientInfo = null; } } void addMessage(StorageApiWritePayload payload) throws Exception { maybeTickleCache(); - if (payload.getSchemaHash() != descriptorWrapper.hash) { - schemaMismatches.inc(); - // The descriptor on the payload doesn't match the descriptor we know about. This - // means that the table has been updated, but that this transform hasn't found out - // about that yet. Refresh the schema and force a new StreamAppendClient to be - // created. - messageConverter.refreshSchema(payload.getSchemaHash()); - descriptorWrapper = messageConverter.getSchemaDescriptor(); - invalidateWriteStream(); - if (useDefaultStream) { - // Since the default stream client is shared across many bundles and threads, we can't - // simply look it up from the cache, as another thread may have recreated it with the - // old - // schema. - getStreamAppendClient(false); - } - // Validate that the record can now be parsed. - DynamicMessage msg = - DynamicMessage.parseFrom(descriptorWrapper.descriptor, payload.getPayload()); - if (msg.getUnknownFields() != null && !msg.getUnknownFields().asMap().isEmpty()) { - throw new RuntimeException( - "Record schema does not match table. Unknown fields: " + msg.getUnknownFields()); - } - } - pendingMessages.add(ByteString.copyFrom(payload.getPayload())); + ByteString payloadBytes = ByteString.copyFrom(payload.getPayload()); + pendingMessages.add(payloadBytes); } long flush( @@ -403,7 +391,8 @@ long flush( for (ByteString rowBytes : inserts.getSerializedRowsList()) { TableRow failedRow = TableRowToStorageApiProto.tableRowFromMessage( - DynamicMessage.parseFrom(descriptorWrapper.descriptor, rowBytes)); + DynamicMessage.parseFrom( + getAppendClientInfo(true, false).descriptor, rowBytes)); failedRowsReceiver.output( new BigQueryStorageApiInsertError( failedRow, "Row payload too large. Maximum size " + maxRequestSize)); @@ -426,7 +415,9 @@ long flush( return ApiFutures.immediateFuture(AppendRowsResponse.newBuilder().build()); } try { - StreamAppendClient writeStream = getStreamAppendClient(true); + StreamAppendClient writeStream = + Preconditions.checkStateNotNull( + getAppendClientInfo(true, true).streamAppendClient); ApiFuture response = writeStream.appendRows(c.offset, c.protoRows); inflightWaitSecondsDistribution.update(writeStream.getInflightWaitSeconds()); @@ -457,7 +448,9 @@ long flush( try { TableRow failedRow = TableRowToStorageApiProto.tableRowFromMessage( - DynamicMessage.parseFrom(descriptorWrapper.descriptor, protoBytes)); + DynamicMessage.parseFrom( + Preconditions.checkStateNotNull(appendClientInfo).descriptor, + protoBytes)); new BigQueryStorageApiInsertError( failedRow, error.getRowIndexToErrorMessage().get(failedIndex)); failedRowsReceiver.output( @@ -622,17 +615,17 @@ DestinationState createDestinationState( MessageConverter messageConverter; try { messageConverter = messageConverters.get(destination, dynamicDestinations, datasetService); + return new DestinationState( + tableDestination1.getTableUrn(), + messageConverter, + datasetService, + useDefaultStream, + streamAppendClientCount, + bigQueryOptions.getUseStorageApiConnectionPool(), + bigQueryOptions.getStorageWriteApiMaxRequestSize()); } catch (Exception e) { throw new RuntimeException(e); } - return new DestinationState( - tableDestination1.getTableUrn(), - messageConverter, - datasetService, - useDefaultStream, - streamAppendClientCount, - bigQueryOptions.getUseStorageApiConnectionPool(), - bigQueryOptions.getStorageWriteApiMaxRequestSize()); } @ProcessElement diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java index af0ae5169bc9..ee3bf69f5036 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java @@ -26,6 +26,7 @@ import com.google.cloud.bigquery.storage.v1.Exceptions; import com.google.cloud.bigquery.storage.v1.Exceptions.StreamFinalizedException; import com.google.cloud.bigquery.storage.v1.ProtoRows; +import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.cloud.bigquery.storage.v1.WriteStream.Type; import com.google.protobuf.ByteString; import com.google.protobuf.DynamicMessage; @@ -40,10 +41,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; @@ -51,8 +52,6 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.StreamAppendClient; import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.RetryType; -import org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations.DescriptorWrapper; -import org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations.MessageConverter; import org.apache.beam.sdk.io.gcp.bigquery.StorageApiFlushAndFinalizeDoFn.Operation; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Distribution; @@ -124,14 +123,45 @@ public class StorageApiWritesShardedRecords> flushTag = new TupleTag<>("flushTag"); private static final ExecutorService closeWriterExecutor = Executors.newCachedThreadPool(); - private static final Cache APPEND_CLIENTS = + // Context passed into RetryManager for each call. + class AppendRowsContext extends RetryManager.Operation.Context { + final ShardedKey key; + String streamName = ""; + @Nullable StreamAppendClient client = null; + long offset = -1; + long numRows = 0; + long tryIteration = 0; + ProtoRows protoRows; + + AppendRowsContext(ShardedKey key, ProtoRows protoRows) { + this.key = key; + this.protoRows = protoRows; + } + + @Override + public String toString() { + return "Context: key=" + + key + + " streamName=" + + streamName + + " offset=" + + offset + + " numRows=" + + numRows + + " tryIteration: " + + tryIteration; + } + }; + + private static final Cache, AppendClientInfo> APPEND_CLIENTS = CacheBuilder.newBuilder() .expireAfterAccess(5, TimeUnit.MINUTES) .removalListener( - (RemovalNotification removal) -> { - final @Nullable StreamAppendClient streamAppendClient = removal.getValue(); - // Close the writer in a different thread so as not to block the main one. - runAsyncIgnoreFailure(closeWriterExecutor, streamAppendClient::close); + (RemovalNotification, AppendClientInfo> removal) -> { + final @Nullable AppendClientInfo appendClientInfo = removal.getValue(); + if (appendClientInfo != null) { + appendClientInfo.close(); + } }) .build(); @@ -175,12 +205,16 @@ public StorageApiWritesShardedRecords( @Override public PCollectionTuple expand( PCollection, Iterable>> input) { + BigQueryOptions bigQueryOptions = input.getPipeline().getOptions().as(BigQueryOptions.class); + final long splitSize = bigQueryOptions.getStorageApiAppendThresholdBytes(); + final long maxRequestSize = bigQueryOptions.getStorageWriteApiMaxRequestSize(); + String operationName = input.getName() + "/" + getName(); // Append records to the Storage API streams. PCollectionTuple writeRecordsResult = input.apply( "Write Records", - ParDo.of(new WriteRecordsDoFn(operationName, streamIdleTime)) + ParDo.of(new WriteRecordsDoFn(operationName, streamIdleTime, splitSize, maxRequestSize)) .withSideInputs(dynamicDestinations.getSideInputs()) .withOutputTags(flushTag, TupleTagList.of(failedRowsTag))); @@ -257,10 +291,15 @@ class WriteRecordsDoFn private final TimerSpec idleTimer = TimerSpecs.timer(TimeDomain.PROCESSING_TIME); private final Duration streamIdleTime; + private final long splitSize; + private final long maxRequestSize; - public WriteRecordsDoFn(String operationName, Duration streamIdleTime) { + public WriteRecordsDoFn( + String operationName, Duration streamIdleTime, long splitSize, long maxRequestSize) { this.messageConverters = new TwoLevelMessageConverterCache<>(operationName); this.streamIdleTime = streamIdleTime; + this.splitSize = splitSize; + this.maxRequestSize = maxRequestSize; } @StartBundle @@ -269,27 +308,29 @@ public void startBundle() throws IOException { } // Get the current stream for this key. If there is no current stream, create one and store the - // stream name in - // persistent state. + // stream name in persistent state. String getOrCreateStream( String tableId, ValueState streamName, ValueState streamOffset, Timer streamIdleTimer, - DatasetService datasetService) - throws IOException, InterruptedException { - String stream = streamName.read(); - if (Strings.isNullOrEmpty(stream)) { - // In a buffered stream, data is only visible up to the offset to which it was flushed. - stream = datasetService.createWriteStream(tableId, Type.BUFFERED).getName(); - streamName.write(stream); - streamOffset.write(0L); - streamsCreated.inc(); - } - // Reset the idle timer. - streamIdleTimer.offset(streamIdleTime).withNoOutputTimestamp().setRelative(); + DatasetService datasetService) { + try { + String stream = streamName.read(); + if (Strings.isNullOrEmpty(stream)) { + // In a buffered stream, data is only visible up to the offset to which it was flushed. + stream = datasetService.createWriteStream(tableId, Type.BUFFERED).getName(); + streamName.write(stream); + streamOffset.write(0L); + streamsCreated.inc(); + } + // Reset the idle timer. + streamIdleTimer.offset(streamIdleTime).withNoOutputTimestamp().setRelative(); - return stream; + return stream; + } catch (Exception e) { + throw new RuntimeException(e); + } } private DatasetService getDatasetService(PipelineOptions pipelineOptions) throws IOException { @@ -340,68 +381,30 @@ public void process( }); final String tableId = tableDestination.getTableUrn(); final DatasetService datasetService = getDatasetService(pipelineOptions); - MessageConverter messageConverter = - messageConverters.get(element.getKey().getKey(), dynamicDestinations, datasetService); - AtomicReference descriptor = - new AtomicReference<>(messageConverter.getSchemaDescriptor()); - - // Each ProtoRows object contains at most 1MB of rows. - // TODO: Push messageFromTableRow up to top level. That we we cans skip TableRow entirely if - // already proto or already schema. - final long splitSize = bigQueryOptions.getStorageApiAppendThresholdBytes(); - // Called if the schema does not match. - Function updateSchemaHash = - (Long expectedHash) -> { - try { - LOG.info("Schema does not match. Querying BigQuery for the current table schema."); - // Update the schema from the table. - messageConverter.refreshSchema(expectedHash); - descriptor.set(messageConverter.getSchemaDescriptor()); - // Force a new connection. - String stream = streamName.read(); - if (stream != null) { - APPEND_CLIENTS.invalidate(stream); - } - return descriptor.get(); - } catch (Exception e) { - throw new RuntimeException(e); - } - }; - Iterable messages = - new SplittingIterable(element.getValue(), splitSize, descriptor.get(), updateSchemaHash); - - class AppendRowsContext extends RetryManager.Operation.Context { - final ShardedKey key; - String streamName = ""; - @Nullable StreamAppendClient client = null; - long offset = -1; - long numRows = 0; - long tryIteration = 0; - ProtoRows protoRows; - - AppendRowsContext(ShardedKey key, ProtoRows protoRows) { - this.key = key; - this.protoRows = protoRows; - } - @Override - public String toString() { - return "Context: key=" - + key - + " streamName=" - + streamName - + " offset=" - + offset - + " numRows=" - + numRows - + " tryIteration: " - + tryIteration; - } - }; + Supplier getOrCreateStream = + () -> getOrCreateStream(tableId, streamName, streamOffset, idleTimer, datasetService); + final AppendClientInfo appendClientInfo = + APPEND_CLIENTS.get( + element.getKey(), + () -> { + @Nullable + TableSchema tableSchema = + messageConverters + .get(element.getKey().getKey(), dynamicDestinations, datasetService) + .getTableSchema(); + return new AppendClientInfo( + tableSchema, + // Make sure that the client is always closed in a different thread to avoid + // blocking. + client -> runAsyncIgnoreFailure(closeWriterExecutor, client::close)) + .createAppendClient(datasetService, getOrCreateStream, false); + }); + + Iterable messages = new SplittingIterable(element.getValue(), splitSize); // Initialize stream names and offsets for all contexts. This will be called initially, but - // will also be called - // if we roll over to a new stream on a retry. + // will also be called if we roll over to a new stream on a retry. BiConsumer, Boolean> initializeContexts = (contexts, isFailure) -> { try { @@ -409,18 +412,13 @@ public String toString() { // Clear the stream name, forcing a new one to be created. streamName.write(""); } - String stream = - getOrCreateStream(tableId, streamName, streamOffset, idleTimer, datasetService); - StreamAppendClient appendClient = - APPEND_CLIENTS.get( - stream, - () -> - datasetService.getStreamAppendClient( - stream, descriptor.get().descriptor, false)); + appendClientInfo.createAppendClient(datasetService, getOrCreateStream, false); + StreamAppendClient streamAppendClient = + Preconditions.checkArgumentNotNull(appendClientInfo.streamAppendClient); for (AppendRowsContext context : contexts) { - context.streamName = stream; - appendClient.pin(); - context.client = appendClient; + context.streamName = streamName.read(); + streamAppendClient.pin(); + context.client = appendClientInfo.streamAppendClient; context.offset = streamOffset.read(); ++context.tryIteration; streamOffset.write(context.offset + context.protoRows.getSerializedRowsCount()); @@ -432,7 +430,7 @@ public String toString() { Consumer> clearClients = contexts -> { - APPEND_CLIENTS.invalidate(streamName.read()); + appendClientInfo.clearAppendClient(); for (AppendRowsContext context : contexts) { if (context.client != null) { // Unpin in a different thread, as it may execute a blocking close. @@ -450,13 +448,9 @@ public String toString() { return ApiFutures.immediateFuture(AppendRowsResponse.newBuilder().build()); } try { - StreamAppendClient appendClient = - APPEND_CLIENTS.get( - context.streamName, - () -> - datasetService.getStreamAppendClient( - context.streamName, descriptor.get().descriptor, false)); - return appendClient.appendRows(context.offset, context.protoRows); + appendClientInfo.createAppendClient(datasetService, getOrCreateStream, false); + return Preconditions.checkStateNotNull(appendClientInfo.streamAppendClient) + .appendRows(context.offset, context.protoRows); } catch (Exception e) { throw new RuntimeException(e); } @@ -485,7 +479,7 @@ public String toString() { try { TableRow failedRow = TableRowToStorageApiProto.tableRowFromMessage( - DynamicMessage.parseFrom(descriptor.get().descriptor, protoBytes)); + DynamicMessage.parseFrom(appendClientInfo.descriptor, protoBytes)); new BigQueryStorageApiInsertError( failedRow, error.getRowIndexToErrorMessage().get(failedIndex)); o.get(failedRowsTag) @@ -580,7 +574,6 @@ public String toString() { false))); flushesScheduled.inc(context.protoRows.getSerializedRowsCount()); }; - long maxRequestSize = bigQueryOptions.getStorageWriteApiMaxRequestSize(); Instant now = Instant.now(); List contexts = Lists.newArrayList(); RetryManager retryManager = @@ -602,7 +595,7 @@ public String toString() { for (ByteString rowBytes : protoRows.getSerializedRowsList()) { TableRow failedRow = TableRowToStorageApiProto.tableRowFromMessage( - DynamicMessage.parseFrom(descriptor.get().descriptor, rowBytes)); + DynamicMessage.parseFrom(appendClientInfo.descriptor, rowBytes)); o.get(failedRowsTag) .output( new BigQueryStorageApiInsertError( @@ -643,6 +636,7 @@ public String toString() { private void finalizeStream( @AlwaysFetched @StateId("streamName") ValueState streamName, @AlwaysFetched @StateId("streamOffset") ValueState streamOffset, + ShardedKey key, MultiOutputReceiver o, org.joda.time.Instant finalizeElementTs) { String stream = MoreObjects.firstNonNull(streamName.read(), ""); @@ -656,12 +650,13 @@ private void finalizeStream( streamName.clear(); streamOffset.clear(); // Make sure that the stream object is closed. - APPEND_CLIENTS.invalidate(stream); + APPEND_CLIENTS.invalidate(key); } } @OnTimer("idleTimer") public void onTimer( + @Key ShardedKey key, @AlwaysFetched @StateId("streamName") ValueState streamName, @AlwaysFetched @StateId("streamOffset") ValueState streamOffset, MultiOutputReceiver o, @@ -672,19 +667,20 @@ public void onTimer( // a pipeline) this finalize element will be dropped as late. This is usually ok as // BigQuery will eventually garbage collect the stream. We attempt to finalize idle streams // merely to remove the pressure of large numbers of orphaned streams from BigQuery. - finalizeStream(streamName, streamOffset, o, window.maxTimestamp()); + finalizeStream(streamName, streamOffset, key, o, window.maxTimestamp()); streamsIdle.inc(); } @OnWindowExpiration public void onWindowExpiration( + @Key ShardedKey key, @AlwaysFetched @StateId("streamName") ValueState streamName, @AlwaysFetched @StateId("streamOffset") ValueState streamOffset, MultiOutputReceiver o, BoundedWindow window) { // Window is done - usually because the pipeline has been drained. Make sure to clean up // streams so that they are not leaked. - finalizeStream(streamName, streamOffset, o, window.maxTimestamp()); + finalizeStream(streamName, streamOffset, key, o, window.maxTimestamp()); } @Override diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java index eaf90fea8295..cb0b0eaa5e3b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java @@ -19,10 +19,10 @@ import static java.util.stream.Collectors.toList; -import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableRow; -import com.google.api.services.bigquery.model.TableSchema; import com.google.cloud.bigquery.storage.v1.BigDecimalByteStringEncoder; +import com.google.cloud.bigquery.storage.v1.TableFieldSchema; +import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.protobuf.ByteString; import com.google.protobuf.DescriptorProtos.DescriptorProto; import com.google.protobuf.DescriptorProtos.FieldDescriptorProto; @@ -56,6 +56,7 @@ import java.util.stream.StreamSupport; import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; @@ -107,30 +108,192 @@ public static class SchemaDoesntMatchException extends SchemaConversionException } } + /////////////////////////////////// + // Conversion between TableSchema the json class and TableSchema the proto class. + + private static final Map MODE_MAP_JSON_PROTO = + ImmutableMap.of( + Mode.NULLABLE, TableFieldSchema.Mode.NULLABLE, + Mode.REQUIRED, TableFieldSchema.Mode.REQUIRED, + Mode.REPEATED, TableFieldSchema.Mode.REPEATED); + private static final Map MODE_MAP_PROTO_JSON = + ImmutableMap.of( + TableFieldSchema.Mode.NULLABLE, "NULLABLE", + TableFieldSchema.Mode.REQUIRED, "REQUIRED", + TableFieldSchema.Mode.REPEATED, "REPEATED"); + + private static final Map TYPE_MAP_JSON_PROTO = + ImmutableMap.builder() + .put("STRUCT", TableFieldSchema.Type.STRUCT) + .put("RECORD", TableFieldSchema.Type.STRUCT) + .put("INT64", TableFieldSchema.Type.INT64) + .put("INTEGER", TableFieldSchema.Type.INT64) + .put("FLOAT64", TableFieldSchema.Type.DOUBLE) + .put("FLOAT", TableFieldSchema.Type.DOUBLE) + .put("STRING", TableFieldSchema.Type.STRING) + .put("BOOL", TableFieldSchema.Type.BOOL) + .put("BOOLEAN", TableFieldSchema.Type.BOOL) + .put("BYTES", TableFieldSchema.Type.BYTES) + .put("NUMERIC", TableFieldSchema.Type.NUMERIC) + .put("BIGNUMERIC", TableFieldSchema.Type.BIGNUMERIC) + .put("GEOGRAPHY", TableFieldSchema.Type.GEOGRAPHY) + .put("DATE", TableFieldSchema.Type.DATE) + .put("TIME", TableFieldSchema.Type.TIME) + .put("DATETIME", TableFieldSchema.Type.DATETIME) + .put("TIMESTAMP", TableFieldSchema.Type.TIMESTAMP) + .put("JSON", TableFieldSchema.Type.JSON) + .build(); + private static final Map TYPE_MAP_PROTO_JSON = + ImmutableMap.builder() + .put(TableFieldSchema.Type.STRUCT, "STRUCT") + .put(TableFieldSchema.Type.INT64, "INT64") + .put(TableFieldSchema.Type.DOUBLE, "FLOAT64") + .put(TableFieldSchema.Type.STRING, "STRING") + .put(TableFieldSchema.Type.BOOL, "BOOL") + .put(TableFieldSchema.Type.BYTES, "BYTES") + .put(TableFieldSchema.Type.NUMERIC, "NUMERIC") + .put(TableFieldSchema.Type.BIGNUMERIC, "BIGNUMERIC") + .put(TableFieldSchema.Type.GEOGRAPHY, "GEOGRAPHY") + .put(TableFieldSchema.Type.DATE, "DATE") + .put(TableFieldSchema.Type.TIME, "TIME") + .put(TableFieldSchema.Type.DATETIME, "DATETIME") + .put(TableFieldSchema.Type.TIMESTAMP, "TIMESTAMP") + .put(TableFieldSchema.Type.JSON, "JSON") + .build(); + + public static TableFieldSchema.Mode modeToProtoMode(String mode) { + return Optional.ofNullable(mode) + .map(Mode::valueOf) + .map(m -> MODE_MAP_JSON_PROTO.get(m)) + .orElse(TableFieldSchema.Mode.REQUIRED); + } + + public static String protoModeToJsonMode(TableFieldSchema.Mode protoMode) { + String jsonMode = MODE_MAP_PROTO_JSON.get(protoMode); + if (jsonMode == null) { + throw new RuntimeException("Unknown mode " + protoMode); + } + return jsonMode; + } + + public static String protoTypeToJsonType(TableFieldSchema.Type protoType) { + String type = TYPE_MAP_PROTO_JSON.get(protoType); + if (type == null) { + throw new RuntimeException("Unknown type " + protoType); + } + return type; + } + + public static TableFieldSchema.Type typeToProtoType(String type) { + TableFieldSchema.Type protoType = TYPE_MAP_JSON_PROTO.get(type); + if (protoType == null) { + throw new RuntimeException("Unknown type " + type); + } + return protoType; + } + + public static com.google.api.services.bigquery.model.TableSchema protoSchemaToTableSchema( + TableSchema protoTableSchema) { + com.google.api.services.bigquery.model.TableSchema tableSchema = + new com.google.api.services.bigquery.model.TableSchema(); + List tableFields = + Lists.newArrayListWithExpectedSize(protoTableSchema.getFieldsCount()); + for (TableFieldSchema protoTableField : protoTableSchema.getFieldsList()) { + tableFields.add(protoTableFieldToTableField(protoTableField)); + } + return tableSchema.setFields(tableFields); + } + + public static com.google.api.services.bigquery.model.TableFieldSchema protoTableFieldToTableField( + TableFieldSchema protoTableField) { + com.google.api.services.bigquery.model.TableFieldSchema tableField = + new com.google.api.services.bigquery.model.TableFieldSchema(); + tableField = tableField.setName(protoTableField.getName()); + if (!Strings.isNullOrEmpty(tableField.getDescription())) { + tableField = tableField.setDescription(protoTableField.getDescription()); + } + if (protoTableField.getMaxLength() != 0) { + tableField = tableField.setMaxLength(protoTableField.getMaxLength()); + } + if (protoTableField.getMode() != TableFieldSchema.Mode.MODE_UNSPECIFIED) { + tableField = tableField.setMode(protoModeToJsonMode(protoTableField.getMode())); + } + if (protoTableField.getPrecision() != 0) { + tableField = tableField.setPrecision(protoTableField.getPrecision()); + } + if (protoTableField.getScale() != 0) { + tableField = tableField.setScale(protoTableField.getScale()); + } + tableField = tableField.setType(protoTypeToJsonType(protoTableField.getType())); + if (protoTableField.getType().equals(TableFieldSchema.Type.STRUCT)) { + List subFields = + Lists.newArrayListWithExpectedSize(protoTableField.getFieldsCount()); + for (TableFieldSchema subField : protoTableField.getFieldsList()) { + subFields.add(protoTableFieldToTableField(subField)); + } + tableField = tableField.setFields(subFields); + } + return tableField; + } + + public static TableSchema schemaToProtoTableSchema( + com.google.api.services.bigquery.model.TableSchema tableSchema) { + TableSchema.Builder builder = TableSchema.newBuilder(); + if (tableSchema.getFields() != null) { + for (com.google.api.services.bigquery.model.TableFieldSchema field : + tableSchema.getFields()) { + builder.addFields(tableFieldToProtoTableField(field)); + } + } + return builder.build(); + } + + public static TableFieldSchema tableFieldToProtoTableField( + com.google.api.services.bigquery.model.TableFieldSchema field) { + TableFieldSchema.Builder builder = TableFieldSchema.newBuilder(); + builder.setName(field.getName()); + if (field.getDescription() != null) { + builder.setDescription(field.getDescription()); + } + if (field.getMaxLength() != null) { + builder.setMaxLength(field.getMaxLength()); + } + if (field.getMode() != null) { + builder.setMode(modeToProtoMode(field.getMode())); + } + if (field.getPrecision() != null) { + builder.setPrecision(field.getPrecision()); + } + if (field.getScale() != null) { + builder.setScale(field.getScale()); + } + builder.setType(typeToProtoType(field.getType())); + if (builder.getType().equals(TableFieldSchema.Type.STRUCT)) { + for (com.google.api.services.bigquery.model.TableFieldSchema subField : field.getFields()) { + builder.addFields(tableFieldToProtoTableField(subField)); + } + } + return builder.build(); + } + static class SchemaInformation { private final TableFieldSchema tableFieldSchema; private final List subFields; private final Map subFieldsByName; private final Iterable parentSchemas; - private SchemaInformation(TableFieldSchema tableFieldSchema) { - this(tableFieldSchema, Collections.emptyList()); - } - private SchemaInformation( TableFieldSchema tableFieldSchema, Iterable parentSchemas) { this.tableFieldSchema = tableFieldSchema; this.subFields = Lists.newArrayList(); this.subFieldsByName = Maps.newHashMap(); this.parentSchemas = parentSchemas; - if (tableFieldSchema.getFields() != null) { - for (TableFieldSchema field : tableFieldSchema.getFields()) { - SchemaInformation schemaInformation = - new SchemaInformation( - field, Iterables.concat(this.parentSchemas, ImmutableList.of(this))); - subFields.add(schemaInformation); - subFieldsByName.put(field.getName(), schemaInformation); - } + for (TableFieldSchema field : tableFieldSchema.getFieldsList()) { + SchemaInformation schemaInformation = + new SchemaInformation( + field, Iterables.concat(this.parentSchemas, ImmutableList.of(this))); + subFields.add(schemaInformation); + subFieldsByName.put(field.getName(), schemaInformation); } } @@ -146,7 +309,7 @@ public String getName() { return tableFieldSchema.getName(); } - public String getType() { + public TableFieldSchema.Type getType() { return tableFieldSchema.getType(); } @@ -167,42 +330,50 @@ public SchemaInformation getSchemaForField(int i) { } static SchemaInformation fromTableSchema(TableSchema tableSchema) { - TableFieldSchema rootSchema = - new TableFieldSchema() - .setName("__root__") - .setType("RECORD") - .setFields(tableSchema.getFields()); - return new SchemaInformation(rootSchema); + TableFieldSchema root = + TableFieldSchema.newBuilder() + .addAllFields(tableSchema.getFieldsList()) + .setName("root") + .build(); + return new SchemaInformation(root, Collections.emptyList()); + } + + static SchemaInformation fromTableSchema( + com.google.api.services.bigquery.model.TableSchema jsonTableSchema) { + return SchemaInformation.fromTableSchema(schemaToProtoTableSchema(jsonTableSchema)); } } - static final Map PRIMITIVE_TYPES = - ImmutableMap.builder() - .put("INT64", Type.TYPE_INT64) - .put("INTEGER", Type.TYPE_INT64) - .put("FLOAT64", Type.TYPE_DOUBLE) - .put("FLOAT", Type.TYPE_DOUBLE) - .put("STRING", Type.TYPE_STRING) - .put("BOOL", Type.TYPE_BOOL) - .put("BOOLEAN", Type.TYPE_BOOL) - .put("BYTES", Type.TYPE_BYTES) - .put("NUMERIC", Type.TYPE_BYTES) - .put("BIGNUMERIC", Type.TYPE_BYTES) - .put("GEOGRAPHY", Type.TYPE_STRING) // Pass through the JSON encoding. - .put("DATE", Type.TYPE_INT32) - .put("TIME", Type.TYPE_INT64) - .put("DATETIME", Type.TYPE_INT64) - .put("TIMESTAMP", Type.TYPE_INT64) - .put("JSON", Type.TYPE_STRING) + static final Map PRIMITIVE_TYPES = + ImmutableMap.builder() + .put(TableFieldSchema.Type.INT64, Type.TYPE_INT64) + .put(TableFieldSchema.Type.DOUBLE, Type.TYPE_DOUBLE) + .put(TableFieldSchema.Type.STRING, Type.TYPE_STRING) + .put(TableFieldSchema.Type.BOOL, Type.TYPE_BOOL) + .put(TableFieldSchema.Type.BYTES, Type.TYPE_BYTES) + .put(TableFieldSchema.Type.NUMERIC, Type.TYPE_BYTES) + .put(TableFieldSchema.Type.BIGNUMERIC, Type.TYPE_BYTES) + .put(TableFieldSchema.Type.GEOGRAPHY, Type.TYPE_STRING) // Pass through the JSON encoding. + .put(TableFieldSchema.Type.DATE, Type.TYPE_INT32) + .put(TableFieldSchema.Type.TIME, Type.TYPE_INT64) + .put(TableFieldSchema.Type.DATETIME, Type.TYPE_INT64) + .put(TableFieldSchema.Type.TIMESTAMP, Type.TYPE_INT64) + .put(TableFieldSchema.Type.JSON, Type.TYPE_STRING) .build(); + public static Descriptor getDescriptorFromTableSchema( + com.google.api.services.bigquery.model.TableSchema jsonSchema, boolean respectRequired) + throws DescriptorValidationException { + return getDescriptorFromTableSchema(schemaToProtoTableSchema(jsonSchema), respectRequired); + } + /** * Given a BigQuery TableSchema, returns a protocol-buffer Descriptor that can be used to write * data using the BigQuery Storage API. */ - public static Descriptor getDescriptorFromTableSchema(TableSchema jsonSchema) - throws DescriptorValidationException { - DescriptorProto descriptorProto = descriptorSchemaFromTableSchema(jsonSchema); + public static Descriptor getDescriptorFromTableSchema( + TableSchema tableSchema, boolean respectRequired) throws DescriptorValidationException { + DescriptorProto descriptorProto = descriptorSchemaFromTableSchema(tableSchema, respectRequired); FileDescriptorProto fileDescriptorProto = FileDescriptorProto.newBuilder().addMessageType(descriptorProto).build(); FileDescriptor fileDescriptor = @@ -218,7 +389,7 @@ public static DynamicMessage messageFromMap( boolean ignoreUnknownValues) throws SchemaConversionException { DynamicMessage.Builder builder = DynamicMessage.newBuilder(descriptor); - for (Map.Entry entry : map.entrySet()) { + for (final Map.Entry entry : map.entrySet()) { @Nullable FieldDescriptor fieldDescriptor = descriptor.findFieldByName(entry.getKey().toLowerCase()); if (fieldDescriptor == null) { @@ -267,7 +438,7 @@ public static DynamicMessage messageFromTableRow( SchemaInformation schemaInformation, Descriptor descriptor, TableRow tableRow, - boolean ignoreUnkownValues) + boolean ignoreUnknownValues) throws SchemaConversionException { @Nullable Object fValue = tableRow.get("f"); if (fValue instanceof List) { @@ -275,13 +446,15 @@ public static DynamicMessage messageFromTableRow( DynamicMessage.Builder builder = DynamicMessage.newBuilder(descriptor); int cellsToProcess = cells.size(); if (cells.size() > descriptor.getFields().size()) { - if (ignoreUnkownValues) { + if (ignoreUnknownValues) { cellsToProcess = descriptor.getFields().size(); } else { throw new SchemaTooNarrowException( - "TableRow contained too many fields and ignoreUnknownValues not set."); + "TableRow contained too many fields and ignoreUnknownValues not set in " + + schemaInformation.getName()); } } + for (int i = 0; i < cellsToProcess; ++i) { AbstractMap cell = cells.get(i); FieldDescriptor fieldDescriptor = descriptor.getFields().get(i); @@ -290,7 +463,7 @@ public static DynamicMessage messageFromTableRow( @Nullable Object value = messageValueFromFieldValue( - fieldSchemaInformation, fieldDescriptor, cell.get("v"), ignoreUnkownValues); + fieldSchemaInformation, fieldDescriptor, cell.get("v"), ignoreUnknownValues); if (value != null) { builder.setField(fieldDescriptor, value); } @@ -311,36 +484,46 @@ public static DynamicMessage messageFromTableRow( "Could convert schema for " + schemaInformation.getFullName(), e); } } else { - return messageFromMap(schemaInformation, descriptor, tableRow, ignoreUnkownValues); + return messageFromMap(schemaInformation, descriptor, tableRow, ignoreUnknownValues); } } @VisibleForTesting - static DescriptorProto descriptorSchemaFromTableSchema(TableSchema tableSchema) { - return descriptorSchemaFromTableFieldSchemas(tableSchema.getFields()); + static DescriptorProto descriptorSchemaFromTableSchema( + com.google.api.services.bigquery.model.TableSchema tableSchema, boolean respectRequired) { + return descriptorSchemaFromTableSchema(schemaToProtoTableSchema(tableSchema), respectRequired); + } + + @VisibleForTesting + static DescriptorProto descriptorSchemaFromTableSchema( + TableSchema tableSchema, boolean respectRequired) { + return descriptorSchemaFromTableFieldSchemas(tableSchema.getFieldsList(), respectRequired); } private static DescriptorProto descriptorSchemaFromTableFieldSchemas( - Iterable tableFieldSchemas) { + Iterable tableFieldSchemas, boolean respectRequired) { DescriptorProto.Builder descriptorBuilder = DescriptorProto.newBuilder(); // Create a unique name for the descriptor ('-' characters cannot be used). descriptorBuilder.setName("D" + UUID.randomUUID().toString().replace("-", "_")); int i = 1; for (TableFieldSchema fieldSchema : tableFieldSchemas) { - fieldDescriptorFromTableField(fieldSchema, i++, descriptorBuilder); + fieldDescriptorFromTableField(fieldSchema, i++, descriptorBuilder, respectRequired); } return descriptorBuilder.build(); } private static void fieldDescriptorFromTableField( - TableFieldSchema fieldSchema, int fieldNumber, DescriptorProto.Builder descriptorBuilder) { + TableFieldSchema fieldSchema, + int fieldNumber, + DescriptorProto.Builder descriptorBuilder, + boolean respectRequired) { FieldDescriptorProto.Builder fieldDescriptorBuilder = FieldDescriptorProto.newBuilder(); fieldDescriptorBuilder = fieldDescriptorBuilder.setName(fieldSchema.getName().toLowerCase()); fieldDescriptorBuilder = fieldDescriptorBuilder.setNumber(fieldNumber); switch (fieldSchema.getType()) { - case "STRUCT": - case "RECORD": - DescriptorProto nested = descriptorSchemaFromTableFieldSchemas(fieldSchema.getFields()); + case STRUCT: + DescriptorProto nested = + descriptorSchemaFromTableFieldSchemas(fieldSchema.getFieldsList(), respectRequired); descriptorBuilder.addNestedType(nested); fieldDescriptorBuilder = fieldDescriptorBuilder.setType(Type.TYPE_MESSAGE).setTypeName(nested.getName()); @@ -354,12 +537,11 @@ private static void fieldDescriptorFromTableField( fieldDescriptorBuilder = fieldDescriptorBuilder.setType(type); } - Optional fieldMode = Optional.ofNullable(fieldSchema.getMode()).map(Mode::valueOf); - if (fieldMode.filter(m -> m == Mode.REPEATED).isPresent()) { + if (fieldSchema.getMode() == TableFieldSchema.Mode.REPEATED) { fieldDescriptorBuilder = fieldDescriptorBuilder.setLabel(Label.LABEL_REPEATED); - } else if (!fieldMode.isPresent() || fieldMode.filter(m -> m == Mode.NULLABLE).isPresent()) { + } else if (!respectRequired || fieldSchema.getMode() == TableFieldSchema.Mode.NULLABLE) { fieldDescriptorBuilder = fieldDescriptorBuilder.setLabel(Label.LABEL_OPTIONAL); - } else { + } else if (fieldSchema.getMode() == TableFieldSchema.Mode.REQUIRED) { fieldDescriptorBuilder = fieldDescriptorBuilder.setLabel(Label.LABEL_REQUIRED); } descriptorBuilder.addField(fieldDescriptorBuilder.build()); @@ -377,6 +559,7 @@ private static void fieldDescriptorFromTableField( } else if (fieldDescriptor.isRepeated()) { return Collections.emptyList(); } else { + // TODO: Allow expanding this! throw new SchemaDoesntMatchException( "Received null value for non-nullable field " + schemaInformation.getFullName()); } @@ -405,31 +588,28 @@ private static void fieldDescriptorFromTableField( boolean ignoreUnknownValues) throws SchemaConversionException { switch (schemaInformation.getType()) { - case "INT64": - case "INTEGER": + case INT64: if (value instanceof String) { return Long.valueOf((String) value); } else if (value instanceof Integer || value instanceof Long) { return ((Number) value).longValue(); } break; - case "FLOAT64": - case "FLOAT": + case DOUBLE: if (value instanceof String) { return Double.valueOf((String) value); } else if (value instanceof Number) { return ((Number) value).doubleValue(); } break; - case "BOOLEAN": - case "BOOL": + case BOOL: if (value instanceof String) { return Boolean.valueOf((String) value); } else if (value instanceof Boolean) { return value; } break; - case "BYTES": + case BYTES: if (value instanceof String) { return ByteString.copyFrom(BaseEncoding.base64().decode((String) value)); } else if (value instanceof byte[]) { @@ -438,7 +618,7 @@ private static void fieldDescriptorFromTableField( return value; } break; - case "TIMESTAMP": + case TIMESTAMP: if (value instanceof String) { try { // '2011-12-03T10:15:30+01:00' '2011-12-03T10:15:30' @@ -470,7 +650,7 @@ private static void fieldDescriptorFromTableField( .longValue(); } break; - case "DATE": + case DATE: if (value instanceof String) { return ((Long) LocalDate.parse((String) value).toEpochDay()).intValue(); } else if (value instanceof LocalDate) { @@ -484,7 +664,7 @@ private static void fieldDescriptorFromTableField( return ((Number) value).intValue(); } break; - case "NUMERIC": + case NUMERIC: if (value instanceof String) { return BigDecimalByteStringEncoder.encodeToNumericByteString( new BigDecimal((String) value)); @@ -495,7 +675,7 @@ private static void fieldDescriptorFromTableField( BigDecimal.valueOf(((Number) value).doubleValue())); } break; - case "BIGNUMERIC": + case BIGNUMERIC: if (value instanceof String) { return BigDecimalByteStringEncoder.encodeToBigNumericByteString( new BigDecimal((String) value)); @@ -506,7 +686,7 @@ private static void fieldDescriptorFromTableField( BigDecimal.valueOf(((Number) value).doubleValue())); } break; - case "DATETIME": + case DATETIME: if (value instanceof String) { try { // '2011-12-03T10:15:30' @@ -525,7 +705,7 @@ private static void fieldDescriptorFromTableField( return CivilTimeEncoder.encodePacked64DatetimeMicros((org.joda.time.LocalDateTime) value); } break; - case "TIME": + case TIME: if (value instanceof String) { return CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.parse((String) value)); } else if (value instanceof Number) { @@ -536,12 +716,11 @@ private static void fieldDescriptorFromTableField( return CivilTimeEncoder.encodePacked64TimeMicros((org.joda.time.LocalTime) value); } break; - case "STRING": - case "JSON": - case "GEOGRAPHY": + case STRING: + case JSON: + case GEOGRAPHY: return Preconditions.checkArgumentNotNull(value).toString(); - case "STRUCT": - case "RECORD": + case STRUCT: if (value instanceof TableRow) { TableRow tableRow = (TableRow) value; return messageFromTableRow( @@ -553,6 +732,8 @@ private static void fieldDescriptorFromTableField( schemaInformation, fieldDescriptor.getMessageType(), map, ignoreUnknownValues); } break; + default: + throw new RuntimeException("Unknown type " + schemaInformation.getType()); } throw new SchemaDoesntMatchException( diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java index 854058a81f35..fefe4d29a2ea 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java @@ -85,21 +85,21 @@ public class BeamRowToStorageApiProtoTest { FieldDescriptorProto.newBuilder() .setName("bytevalue") .setNumber(1) - .setType(Type.TYPE_INT32) + .setType(Type.TYPE_INT64) .setLabel(Label.LABEL_OPTIONAL) .build()) .addField( FieldDescriptorProto.newBuilder() .setName("int16value") .setNumber(2) - .setType(Type.TYPE_INT32) + .setType(Type.TYPE_INT64) .setLabel(Label.LABEL_REQUIRED) .build()) .addField( FieldDescriptorProto.newBuilder() .setName("int32value") .setNumber(3) - .setType(Type.TYPE_INT32) + .setType(Type.TYPE_INT64) .setLabel(Label.LABEL_OPTIONAL) .build()) .addField( @@ -120,7 +120,7 @@ public class BeamRowToStorageApiProtoTest { FieldDescriptorProto.newBuilder() .setName("floatvalue") .setNumber(6) - .setType(Type.TYPE_FLOAT) + .setType(Type.TYPE_DOUBLE) .setLabel(Label.LABEL_OPTIONAL) .build()) .addField( @@ -233,14 +233,14 @@ public class BeamRowToStorageApiProtoTest { .build(); private static final Map BASE_PROTO_EXPECTED_FIELDS = ImmutableMap.builder() - .put("bytevalue", (int) 1) - .put("int16value", (int) 2) - .put("int32value", (int) 3) - .put("int64value", (long) 4) + .put("bytevalue", 1L) + .put("int16value", 2L) + .put("int32value", 3L) + .put("int64value", 4L) .put( "decimalvalue", BeamRowToStorageApiProto.serializeBigDecimalToNumeric(BigDecimal.valueOf(5))) - .put("floatvalue", (float) 3.14) + .put("floatvalue", (double) 3.14) .put("doublevalue", (double) 2.68) .put("stringvalue", "I am a string. Hear me roar.") .put("datetimevalue", BASE_ROW.getDateTime("datetimeValue").getMillis() * 1000) @@ -284,7 +284,8 @@ public class BeamRowToStorageApiProtoTest { @Test public void testDescriptorFromSchema() { DescriptorProto descriptor = - BeamRowToStorageApiProto.descriptorSchemaFromBeamSchema(BASE_SCHEMA); + TableRowToStorageApiProto.descriptorSchemaFromTableSchema( + BeamRowToStorageApiProto.protoTableSchemaFromBeamSchema(BASE_SCHEMA), true); Map types = descriptor.getFieldList().stream() .collect( @@ -315,7 +316,8 @@ public void testDescriptorFromSchema() { @Test public void testNestedFromSchema() { DescriptorProto descriptor = - BeamRowToStorageApiProto.descriptorSchemaFromBeamSchema(NESTED_SCHEMA); + TableRowToStorageApiProto.descriptorSchemaFromTableSchema( + BeamRowToStorageApiProto.protoTableSchemaFromBeamSchema((NESTED_SCHEMA)), true); Map expectedBaseTypes = BASE_SCHEMA_PROTO.getFieldList().stream() .collect( @@ -378,7 +380,9 @@ private void assertBaseRecord(DynamicMessage msg) { @Test public void testMessageFromTableRow() throws Exception { - Descriptor descriptor = BeamRowToStorageApiProto.getDescriptorFromSchema(NESTED_SCHEMA); + Descriptor descriptor = + TableRowToStorageApiProto.getDescriptorFromTableSchema( + BeamRowToStorageApiProto.protoTableSchemaFromBeamSchema(NESTED_SCHEMA), true); DynamicMessage msg = BeamRowToStorageApiProto.messageFromBeamRow(descriptor, NESTED_ROW); assertEquals(3, msg.getAllFields().size()); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java index 1e1749e8569a..9cacfde54adc 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java @@ -33,6 +33,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; import com.google.api.services.bigquery.model.Clustering; import com.google.api.services.bigquery.model.ErrorProto; @@ -65,11 +66,9 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Function; -import java.util.function.LongFunction; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.LongStream; import java.util.stream.StreamSupport; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericDatumWriter; @@ -199,7 +198,6 @@ public Statement apply(final Statement base, final Description description) { public void evaluate() throws Throwable { options = TestPipeline.testingPipelineOptions(); BigQueryOptions bqOptions = options.as(BigQueryOptions.class); - bqOptions.setSchemaUpdateRetries(Integer.MAX_VALUE); bqOptions.setProject("project-id"); if (description.getAnnotations().stream() .anyMatch(a -> a.annotationType().equals(ProjectOverride.class))) { @@ -254,9 +252,8 @@ abstract static class StringLongDestinations extends DynamicDestinations elements = Lists.newArrayList(); for (int i = 0; i < 30; ++i) { elements.add(new TableRow().set("number", i)); @@ -677,9 +669,8 @@ public void testTriggeredFileLoadsWithTempTablesAndDataset() throws Exception { } public void testTriggeredFileLoadsWithTempTables(String tableRef) throws Exception { - if (useStorageApi || !useStreaming) { - return; - } + assumeTrue(!useStorageApi); + assumeTrue(useStreaming); List elements = Lists.newArrayList(); for (int i = 0; i < 30; ++i) { elements.add(new TableRow().set("number", i)); @@ -739,9 +730,8 @@ public void testTriggeredFileLoadsWithTempTables() throws Exception { @Test public void testUntriggeredFileLoadsWithTempTables() throws Exception { // Test only non-streaming inserts. - if (useStorageApi || useStreaming) { - return; - } + assumeTrue(!useStorageApi); + assumeTrue(!useStreaming); List elements = Lists.newArrayList(); for (int i = 0; i < 30; ++i) { elements.add(new TableRow().set("number", i)); @@ -773,10 +763,8 @@ public void testTriggeredFileLoadsWithTempTablesDefaultProject() throws Exceptio @Test public void testTriggeredFileLoadsWithAutoSharding() throws Exception { - if (useStorageApi || !useStreaming) { - // This test does not make sense for the storage API. - return; - } + assumeTrue(!useStorageApi); + assumeTrue(useStreaming); List elements = Lists.newArrayList(); for (int i = 0; i < 30; ++i) { elements.add(new TableRow().set("number", i)); @@ -845,9 +833,8 @@ public void testTriggeredFileLoadsWithAutoSharding() throws Exception { @Test public void testFailuresNoRetryPolicy() throws Exception { - if (useStorageApi || !useStreaming) { - return; - } + assumeTrue(!useStorageApi); + assumeTrue(useStreaming); TableRow row1 = new TableRow().set("name", "a").set("number", "1"); TableRow row2 = new TableRow().set("name", "b").set("number", "2"); TableRow row3 = new TableRow().set("name", "c").set("number", "3"); @@ -884,9 +871,8 @@ public void testFailuresNoRetryPolicy() throws Exception { @Test public void testRetryPolicy() throws Exception { - if (useStorageApi || !useStreaming) { - return; - } + assumeTrue(!useStorageApi); + assumeTrue(useStreaming); TableRow row1 = new TableRow().set("name", "a").set("number", "1"); TableRow row2 = new TableRow().set("name", "b").set("number", "2"); TableRow row3 = new TableRow().set("name", "c").set("number", "3"); @@ -960,9 +946,8 @@ public void testWrite() throws Exception { @Test public void testWriteWithSuccessfulBatchInserts() throws Exception { - if (useStreaming || useStorageApi) { - return; - } + assumeTrue(!useStreaming); + assumeTrue(!useStorageApi); WriteResult result = p.apply( @@ -992,9 +977,8 @@ public void testWriteWithSuccessfulBatchInserts() throws Exception { @Test public void testWriteWithSuccessfulBatchInsertsAndWriteRename() throws Exception { - if (useStreaming || useStorageApi) { - return; - } + assumeTrue(!useStreaming); + assumeTrue(!useStorageApi); WriteResult result = p.apply( @@ -1026,9 +1010,8 @@ public void testWriteWithSuccessfulBatchInsertsAndWriteRename() throws Exception @Test public void testWriteWithoutInsertId() throws Exception { - if (useStorageApi || !useStreaming) { - return; - } + assumeTrue(!useStorageApi); + assumeTrue(useStreaming); TableRow row1 = new TableRow().set("name", "a").set("number", 1); TableRow row2 = new TableRow().set("name", "b").set("number", 2); TableRow row3 = new TableRow().set("name", "c").set("number", 3); @@ -1079,9 +1062,9 @@ public static InputRecord create( @Test public void testWriteAvro() throws Exception { - if (useStorageApi || useStreaming) { - return; - } + assumeTrue(!useStorageApi); + assumeTrue(!useStreaming); + p.apply( Create.of( InputRecord.create("test", 1, 1.0, Instant.parse("2019-01-01T00:00:00Z")), @@ -1130,9 +1113,8 @@ public void testWriteAvro() throws Exception { @Test public void testWriteAvroWithCustomWriter() throws Exception { - if (useStorageApi || useStreaming) { - return; - } + assumeTrue(!useStorageApi); + assumeTrue(!useStreaming); SerializableFunction, GenericRecord> formatFunction = r -> { GenericRecord rec = new GenericData.Record(r.getSchema()); @@ -1246,9 +1228,7 @@ public void testStorageApiWriteWithAutoSharding() throws Exception { } private void storageWrite(boolean autoSharding) throws Exception { - if (!useStorageApi) { - return; - } + assumeTrue(useStorageApi); BigQueryIO.Write write = BigQueryIO.writeTableRows() .to("project-id:dataset-id.table-id") @@ -1302,6 +1282,7 @@ static class SchemaPojo { @Test public void testSchemaWriteLoads() throws Exception { + assumeTrue(!useStreaming); // withMethod overrides the pipeline option, so we need to explicitly request // STORAGE_API_WRITES. BigQueryIO.Write.Method method = @@ -1329,17 +1310,17 @@ public void testSchemaWriteLoads() throws Exception { assertThat( fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"), containsInAnyOrder( - new TableRow().set("name", "a").set("number", 1), - new TableRow().set("name", "b").set("number", 2), - new TableRow().set("name", "c").set("number", 3), - new TableRow().set("name", "d").set("number", 4))); + new TableRow().set("name", "a").set("number", "1"), + new TableRow().set("name", "b").set("number", "2"), + new TableRow().set("name", "c").set("number", "3"), + new TableRow().set("name", "d").set("number", "4"))); } @Test public void testSchemaWriteStreams() throws Exception { - if (useStorageApi || !useStreaming) { - return; - } + assumeTrue(!useStorageApi); + assumeTrue(useStreaming); + WriteResult result = p.apply( Create.of( @@ -1358,22 +1339,20 @@ public void testSchemaWriteStreams() throws Exception { PAssert.that(result.getSuccessfulInserts()) .satisfies( - new SerializableFunction, Void>() { - @Override - public Void apply(Iterable input) { - assertThat(Lists.newArrayList(input).size(), is(4)); - return null; - } - }); + (SerializableFunction, Void>) + input -> { + assertThat(Lists.newArrayList(input).size(), is(4)); + return null; + }); p.run(); assertThat( fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"), containsInAnyOrder( - new TableRow().set("name", "a").set("number", 1), - new TableRow().set("name", "b").set("number", 2), - new TableRow().set("name", "c").set("number", 3), - new TableRow().set("name", "d").set("number", 4))); + new TableRow().set("name", "a").set("number", "1"), + new TableRow().set("name", "b").set("number", "2"), + new TableRow().set("name", "c").set("number", "3"), + new TableRow().set("name", "d").set("number", "4"))); } /** @@ -1572,9 +1551,7 @@ public void testWriteWithDynamicTables() throws Exception { @Test public void testWriteUnknown() throws Exception { - if (useStorageApi) { - return; - } + assumeTrue(!useStorageApi); p.apply( Create.of( new TableRow().set("name", "a").set("number", 1), @@ -1595,9 +1572,7 @@ public void testWriteUnknown() throws Exception { @Test public void testWriteFailedJobs() throws Exception { - if (useStorageApi) { - return; - } + assumeTrue(!useStorageApi); p.apply( Create.of( new TableRow().set("name", "a").set("number", 1), @@ -1819,120 +1794,6 @@ public TableRow apply(Long input) { p.run(); } - @Test - public void testUpdateTableSchemaUseSet() throws Exception { - updateTableSchemaTest(true); - } - - @Test - public void testUpdateTableSchemaUseSetF() throws Exception { - updateTableSchemaTest(false); - } - - public void updateTableSchemaTest(boolean useSet) throws Exception { - if (!useStreaming || !useStorageApi) { - return; - } - BigQueryIO.Write.Method method = - useStorageApiApproximate ? Method.STORAGE_API_AT_LEAST_ONCE : Method.STORAGE_WRITE_API; - p.enableAbandonedNodeEnforcement(false); - - TableReference tableRef = BigQueryHelpers.parseTableSpec("project-id:dataset-id.table"); - TableSchema tableSchema = - new TableSchema() - .setFields( - ImmutableList.of( - new TableFieldSchema().setName("name").setType("STRING"), - new TableFieldSchema().setName("number").setType("INTEGER"))); - fakeDatasetService.createTable(new Table().setTableReference(tableRef).setSchema(tableSchema)); - - LongFunction getRowSet = - (LongFunction & Serializable) - (long i) -> { - if (i < 5) { - return new TableRow().set("name", "name" + i).set("number", Long.toString(i)); - - } else { - return new TableRow() - .set("name", "name" + i) - .set("number", Long.toString(i)) - .set("double_number", Long.toString(i * 2)); - } - }; - - LongFunction getRowSetF = - (LongFunction & Serializable) - (long i) -> { - if (i < 5) { - return new TableRow() - .setF( - ImmutableList.of( - new TableCell().setV("name" + i), - new TableCell().setV(Long.toString(i)))); - } else { - return new TableRow() - .setF( - ImmutableList.of( - new TableCell().setV("name" + i), - new TableCell().setV(Long.toString(i)), - new TableCell().setV(Long.toString(i * 2)))); - } - }; - - LongFunction getRow = useSet ? getRowSet : getRowSetF; - final int numRows = 10; - PCollection tableRows = - p.apply(GenerateSequence.from(0).to(numRows)) - .apply( - MapElements.via( - new SimpleFunction() { - @Override - public TableRow apply(Long input) { - return getRow.apply(input); - } - })) - .setCoder(TableRowJsonCoder.of()); - tableRows.apply( - BigQueryIO.writeTableRows() - .to(tableRef) - .withMethod(method) - .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER) - .withAutoSchemaUpdate(true) - .withTestServices(fakeBqServices) - .withoutValidation()); - - Thread thread = - new Thread( - () -> { - try { - Thread.sleep(5000); - TableSchema tableSchemaUpdated = - new TableSchema() - .setFields( - ImmutableList.of( - new TableFieldSchema().setName("name").setType("STRING"), - new TableFieldSchema().setName("number").setType("INTEGER"), - new TableFieldSchema() - .setName("double_number") - .setType("INTEGER"))); - fakeDatasetService.updateTableSchema(tableRef, tableSchemaUpdated); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - thread.start(); - p.run(); - thread.join(); - - assertThat( - fakeDatasetService.getAllRows( - tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId()), - containsInAnyOrder( - Iterables.toArray( - LongStream.range(0, numRows).mapToObj(getRowSet).collect(Collectors.toList()), - TableRow.class))); - } - @Test public void testBigQueryIOGetName() { assertEquals( @@ -1969,9 +1830,7 @@ public void testWriteValidateFailsNoFormatFunction() { @Test public void testWriteValidateFailsBothFormatFunctions() { - if (useStorageApi) { - return; - } + assumeTrue(!useStorageApi); p.enableAbandonedNodeEnforcement(false); thrown.expect(IllegalArgumentException.class); @@ -1990,9 +1849,7 @@ public void testWriteValidateFailsBothFormatFunctions() { @Test public void testWriteValidateFailsWithBeamSchemaAndAvroFormatFunction() { - if (useStorageApi) { - return; - } + assumeTrue(!useStorageApi); p.enableAbandonedNodeEnforcement(false); thrown.expect(IllegalArgumentException.class); @@ -2008,9 +1865,8 @@ public void testWriteValidateFailsWithBeamSchemaAndAvroFormatFunction() { @Test public void testWriteValidateFailsWithAvroFormatAndStreamingInserts() { - if (!useStreaming && !useStorageApi) { - return; - } + assumeTrue(useStreaming); + assumeTrue(!useStorageApi); p.enableAbandonedNodeEnforcement(false); thrown.expect(IllegalArgumentException.class); @@ -2027,9 +1883,7 @@ public void testWriteValidateFailsWithAvroFormatAndStreamingInserts() { @Test public void testWriteValidateFailsWithBatchAutoSharding() { - if (useStorageApi) { - return; - } + assumeTrue(!useStorageApi); p.enableAbandonedNodeEnforcement(false); thrown.expect(IllegalArgumentException.class); @@ -2497,9 +2351,7 @@ public void testWriteToTableDecorator() throws Exception { @Test public void testExtendedErrorRetrieval() throws Exception { - if (useStorageApi) { - return; - } + assumeTrue(!useStorageApi); TableRow row1 = new TableRow().set("name", "a").set("number", "1"); TableRow row2 = new TableRow().set("name", "b").set("number", "2"); TableRow row3 = new TableRow().set("name", "c").set("number", "3"); @@ -2552,13 +2404,7 @@ public void testExtendedErrorRetrieval() throws Exception { @Test public void testStorageApiErrors() throws Exception { - BigQueryOptions bqOptions = p.getOptions().as(BigQueryOptions.class); - bqOptions.setSchemaUpdateRetries(1); - - if (!useStorageApi) { - return; - } - + assumeTrue(useStorageApi); final Method method = useStorageApiApproximate ? Method.STORAGE_API_AT_LEAST_ONCE : Method.STORAGE_WRITE_API; @@ -2657,9 +2503,7 @@ public void testStorageApiErrors() throws Exception { @Test public void testWrongErrorConfigs() { - if (useStorageApi) { - return; - } + assumeTrue(!useStorageApi); p.enableAutoRunIfMissing(true); TableRow row1 = new TableRow().set("name", "a").set("number", "1"); @@ -2777,9 +2621,8 @@ public void testWriteFileSchemaUpdateOptionAll() throws Exception { @Test public void testSchemaUpdateOptionsFailsStreamingInserts() throws Exception { - if (!useStreaming && !useStorageApi) { - return; - } + assumeTrue(useStreaming); + assumeTrue(!useStorageApi); Set options = EnumSet.of(SchemaUpdateOption.ALLOW_FIELD_ADDITION); p.enableAbandonedNodeEnforcement(false); thrown.expect(IllegalArgumentException.class); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java index b832a9b3612e..eacb95a9a683 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java @@ -599,7 +599,7 @@ public void testToTableRow_flat() { assertThat(row.size(), equalTo(22)); assertThat(row, hasEntry("id", "123")); - assertThat(row, hasEntry("value", 123.456)); + assertThat(row, hasEntry("value", "123.456")); assertThat(row, hasEntry("datetime", "2020-11-02T12:34:56.789876")); assertThat(row, hasEntry("datetime0ms", "2020-11-02T12:34:56")); assertThat(row, hasEntry("datetime0s_ns", "2020-11-02T12:34:00.789876")); @@ -610,12 +610,12 @@ public void testToTableRow_flat() { assertThat(row, hasEntry("time0s_ns", "12:34:00.789876")); assertThat(row, hasEntry("time0s_0ns", "12:34:00")); assertThat(row, hasEntry("name", "test")); - assertThat(row, hasEntry("valid", false)); + assertThat(row, hasEntry("valid", "false")); assertThat(row, hasEntry("binary", "ABCD1234")); assertThat(row, hasEntry("numeric", "123.456")); - assertThat(row, hasEntry("boolean", true)); + assertThat(row, hasEntry("boolean", "true")); assertThat(row, hasEntry("long", "123")); - assertThat(row, hasEntry("double", 123.456)); + assertThat(row, hasEntry("double", "123.456")); } @Test @@ -642,7 +642,7 @@ public void testToTableRow_map() { row = ((List) row.get("map")).get(0); assertThat(row.size(), equalTo(2)); assertThat(row, hasEntry("key", "test")); - assertThat(row, hasEntry("value", 123.456)); + assertThat(row, hasEntry("value", "123.456")); } @Test @@ -653,7 +653,7 @@ public void testToTableRow_row() { row = (TableRow) row.get("row"); assertThat(row.size(), equalTo(22)); assertThat(row, hasEntry("id", "123")); - assertThat(row, hasEntry("value", 123.456)); + assertThat(row, hasEntry("value", "123.456")); assertThat(row, hasEntry("datetime", "2020-11-02T12:34:56.789876")); assertThat(row, hasEntry("datetime0ms", "2020-11-02T12:34:56")); assertThat(row, hasEntry("datetime0s_ns", "2020-11-02T12:34:00.789876")); @@ -664,12 +664,12 @@ public void testToTableRow_row() { assertThat(row, hasEntry("time0s_ns", "12:34:00.789876")); assertThat(row, hasEntry("time0s_0ns", "12:34:00")); assertThat(row, hasEntry("name", "test")); - assertThat(row, hasEntry("valid", false)); + assertThat(row, hasEntry("valid", "false")); assertThat(row, hasEntry("binary", "ABCD1234")); assertThat(row, hasEntry("numeric", "123.456")); - assertThat(row, hasEntry("boolean", true)); + assertThat(row, hasEntry("boolean", "true")); assertThat(row, hasEntry("long", "123")); - assertThat(row, hasEntry("double", 123.456)); + assertThat(row, hasEntry("double", "123.456")); } @Test @@ -680,7 +680,7 @@ public void testToTableRow_array_row() { row = ((List) row.get("rows")).get(0); assertThat(row.size(), equalTo(22)); assertThat(row, hasEntry("id", "123")); - assertThat(row, hasEntry("value", 123.456)); + assertThat(row, hasEntry("value", "123.456")); assertThat(row, hasEntry("datetime", "2020-11-02T12:34:56.789876")); assertThat(row, hasEntry("datetime0ms", "2020-11-02T12:34:56")); assertThat(row, hasEntry("datetime0s_ns", "2020-11-02T12:34:00.789876")); @@ -691,12 +691,12 @@ public void testToTableRow_array_row() { assertThat(row, hasEntry("time0s_ns", "12:34:00.789876")); assertThat(row, hasEntry("time0s_0ns", "12:34:00")); assertThat(row, hasEntry("name", "test")); - assertThat(row, hasEntry("valid", false)); + assertThat(row, hasEntry("valid", "false")); assertThat(row, hasEntry("binary", "ABCD1234")); assertThat(row, hasEntry("numeric", "123.456")); - assertThat(row, hasEntry("boolean", true)); + assertThat(row, hasEntry("boolean", "true")); assertThat(row, hasEntry("long", "123")); - assertThat(row, hasEntry("double", 123.456)); + assertThat(row, hasEntry("double", "123.456")); } @Test diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java index 0a376c8082d3..27a772a60edf 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java @@ -44,7 +44,9 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -467,10 +469,12 @@ public class TableRowToStorageApiProtoTest { .setFields(BASE_TABLE_SCHEMA_NO_F.getFields())) .build()); + @Rule public transient ExpectedException thrown = ExpectedException.none(); + @Test public void testDescriptorFromTableSchema() { DescriptorProto descriptor = - TableRowToStorageApiProto.descriptorSchemaFromTableSchema(BASE_TABLE_SCHEMA); + TableRowToStorageApiProto.descriptorSchemaFromTableSchema(BASE_TABLE_SCHEMA, true); Map types = descriptor.getFieldList().stream() .collect( @@ -485,7 +489,7 @@ public void testDescriptorFromTableSchema() { @Test public void testNestedFromTableSchema() { DescriptorProto descriptor = - TableRowToStorageApiProto.descriptorSchemaFromTableSchema(NESTED_TABLE_SCHEMA); + TableRowToStorageApiProto.descriptorSchemaFromTableSchema(NESTED_TABLE_SCHEMA, true); Map expectedBaseTypes = BASE_TABLE_SCHEMA_PROTO.getFieldList().stream() .collect( @@ -687,7 +691,7 @@ public void testMessageFromTableRow() throws Exception { .set("nestedValueNoF2", BASE_TABLE_ROW_NO_F); Descriptor descriptor = - TableRowToStorageApiProto.getDescriptorFromTableSchema(NESTED_TABLE_SCHEMA); + TableRowToStorageApiProto.getDescriptorFromTableSchema(NESTED_TABLE_SCHEMA, true); TableRowToStorageApiProto.SchemaInformation schemaInformation = TableRowToStorageApiProto.SchemaInformation.fromTableSchema(NESTED_TABLE_SCHEMA); DynamicMessage msg = @@ -707,7 +711,7 @@ public void testMessageFromTableRow() throws Exception { @Test public void testMessageWithFFromTableRow() throws Exception { Descriptor descriptor = - TableRowToStorageApiProto.getDescriptorFromTableSchema(BASE_TABLE_SCHEMA); + TableRowToStorageApiProto.getDescriptorFromTableSchema(BASE_TABLE_SCHEMA, true); TableRowToStorageApiProto.SchemaInformation schemaInformation = TableRowToStorageApiProto.SchemaInformation.fromTableSchema(BASE_TABLE_SCHEMA); DynamicMessage msg = @@ -750,7 +754,7 @@ public void testRepeatedDescriptorFromTableSchema() throws Exception { .set("repeatednof1", ImmutableList.of(BASE_TABLE_ROW_NO_F, BASE_TABLE_ROW_NO_F)) .set("repeatednof2", ImmutableList.of(BASE_TABLE_ROW_NO_F, BASE_TABLE_ROW_NO_F)); Descriptor descriptor = - TableRowToStorageApiProto.getDescriptorFromTableSchema(REPEATED_MESSAGE_SCHEMA); + TableRowToStorageApiProto.getDescriptorFromTableSchema(REPEATED_MESSAGE_SCHEMA, true); TableRowToStorageApiProto.SchemaInformation schemaInformation = TableRowToStorageApiProto.SchemaInformation.fromTableSchema(REPEATED_MESSAGE_SCHEMA); DynamicMessage msg = @@ -795,7 +799,7 @@ public void testNullRepeatedDescriptorFromTableSchema() throws Exception { .set("repeatednof1", null) .set("repeatednof2", null); Descriptor descriptor = - TableRowToStorageApiProto.getDescriptorFromTableSchema(REPEATED_MESSAGE_SCHEMA); + TableRowToStorageApiProto.getDescriptorFromTableSchema(REPEATED_MESSAGE_SCHEMA, true); TableRowToStorageApiProto.SchemaInformation schemaInformation = TableRowToStorageApiProto.SchemaInformation.fromTableSchema(REPEATED_MESSAGE_SCHEMA); DynamicMessage msg = @@ -818,4 +822,70 @@ public void testNullRepeatedDescriptorFromTableSchema() throws Exception { (List) msg.getField(fieldDescriptors.get("repeatednof2")); assertTrue(repeatednof2.isEmpty()); } + + @Test + public void testRejectUnknownField() throws Exception { + TableRow row = new TableRow(); + row.putAll(BASE_TABLE_ROW_NO_F); + row.set("unknown", "foobar"); + + Descriptor descriptor = + TableRowToStorageApiProto.getDescriptorFromTableSchema(BASE_TABLE_SCHEMA_NO_F, true); + TableRowToStorageApiProto.SchemaInformation schemaInformation = + TableRowToStorageApiProto.SchemaInformation.fromTableSchema(BASE_TABLE_SCHEMA_NO_F); + + thrown.expect(TableRowToStorageApiProto.SchemaConversionException.class); + TableRowToStorageApiProto.messageFromTableRow(schemaInformation, descriptor, row, false); + } + + @Test + public void testRejectUnknownFieldF() throws Exception { + TableRow row = new TableRow(); + List cells = Lists.newArrayList(BASE_TABLE_ROW.getF()); + cells.add(new TableCell().setV("foobar")); + row.setF(cells); + + Descriptor descriptor = + TableRowToStorageApiProto.getDescriptorFromTableSchema(BASE_TABLE_SCHEMA, true); + TableRowToStorageApiProto.SchemaInformation schemaInformation = + TableRowToStorageApiProto.SchemaInformation.fromTableSchema(BASE_TABLE_SCHEMA); + + thrown.expect(TableRowToStorageApiProto.SchemaConversionException.class); + TableRowToStorageApiProto.messageFromTableRow(schemaInformation, descriptor, row, false); + } + + @Test + public void testRejectUnknownNestedField() throws Exception { + TableRow rowNoF = new TableRow(); + rowNoF.putAll(BASE_TABLE_ROW_NO_F); + rowNoF.set("unknown", "foobar"); + + TableRow topRow = new TableRow().set("nestedValueNoF1", rowNoF); + + Descriptor descriptor = + TableRowToStorageApiProto.getDescriptorFromTableSchema(NESTED_TABLE_SCHEMA, true); + TableRowToStorageApiProto.SchemaInformation schemaInformation = + TableRowToStorageApiProto.SchemaInformation.fromTableSchema(NESTED_TABLE_SCHEMA); + + thrown.expect(TableRowToStorageApiProto.SchemaConversionException.class); + TableRowToStorageApiProto.messageFromTableRow(schemaInformation, descriptor, topRow, false); + } + + @Test + public void testRejectUnknownNestedFieldF() throws Exception { + TableRow rowWithF = new TableRow(); + List cells = Lists.newArrayList(BASE_TABLE_ROW.getF()); + cells.add(new TableCell().setV("foobar")); + rowWithF.setF(cells); + + TableRow topRow = new TableRow().set("nestedValue1", rowWithF); + + Descriptor descriptor = + TableRowToStorageApiProto.getDescriptorFromTableSchema(NESTED_TABLE_SCHEMA, true); + TableRowToStorageApiProto.SchemaInformation schemaInformation = + TableRowToStorageApiProto.SchemaInformation.fromTableSchema(NESTED_TABLE_SCHEMA); + + thrown.expect(TableRowToStorageApiProto.SchemaConversionException.class); + TableRowToStorageApiProto.messageFromTableRow(schemaInformation, descriptor, topRow, false); + } } From 671fbdac424f2bfac6eccee472041af4d3f0d5d4 Mon Sep 17 00:00:00 2001 From: Andrew Pilloud Date: Thu, 17 Nov 2022 15:39:14 -0800 Subject: [PATCH 262/269] Simplify sdks/java/harness build This is a noop change, moves things closer to legacy-worker/build.gradle --- sdks/java/harness/build.gradle | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/sdks/java/harness/build.gradle b/sdks/java/harness/build.gradle index c179dee5c280..74c5e897edc7 100644 --- a/sdks/java/harness/build.gradle +++ b/sdks/java/harness/build.gradle @@ -18,13 +18,6 @@ plugins { id 'org.apache.beam.module' } -// We specifically enumerate all the projects that we depend on since -// the list is used in both defining the included set for the uber jar -// and also the set of project level dependencies. -def dependOnShadedProjects = [":model:pipeline", ":model:fn-execution", ":sdks:java:core"] -def dependOnProjects = [":sdks:java:fn-execution", - ":runners:core-java", ":runners:core-construction-java"] - applyJavaNature( classesTriggerCheckerBugs: [ 'AssignWindowsRunner': 'https://github.com/typetools/checker-framework/issues/3794', @@ -40,12 +33,6 @@ applyJavaNature( // jars. { dependencies { - // Directly include all projects depended on - dependOnProjects.each { - include(project(path: it, configuration: "shadow")) - } - // Include all dependencies and transitive dependencies - include(dependency(".*:.*")) exclude(dependency(".*:avro:.*")) } }, @@ -55,21 +42,20 @@ description = "Apache Beam :: SDKs :: Java :: Harness" ext.summary = "This contains the SDK Fn Harness for Beam Java" dependencies { - dependOnShadedProjects.each { - implementation project(path: it, configuration: "shadow") - } - dependOnProjects.each { - implementation project(it) - } - shadow library.java.vendored_guava_26_0_jre + implementation project(path: ":model:fn-execution", configuration: "shadow") + implementation project(path: ":model:pipeline", configuration: "shadow") + implementation project(path: ":sdks:java:core", configuration: "shadow") + implementation project(":runners:core-construction-java") + implementation project(":runners:core-java") + implementation project(":sdks:java:fn-execution") + implementation library.java.jamm implementation library.java.joda_time implementation library.java.slf4j_api implementation library.java.vendored_grpc_1_48_1 - implementation library.java.jamm + shadow library.java.vendored_guava_26_0_jre testImplementation library.java.junit testImplementation library.java.mockito_core testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") - testImplementation project(":runners:core-construction-java") testImplementation project(path: ":sdks:java:fn-execution", configuration: "testRuntimeMigration") shadowTestRuntimeClasspath library.java.slf4j_jdk14 } From 14d6cd77589294efdd8124a58d6df4bed3d21d18 Mon Sep 17 00:00:00 2001 From: Andrew Pilloud Date: Thu, 17 Nov 2022 17:38:30 -0800 Subject: [PATCH 263/269] Move configuration changes before shadowJar --- .../beam/gradle/BeamModulePlugin.groovy | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 107fbefd164b..b1488de0badb 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1028,6 +1028,50 @@ class BeamModulePlugin implements Plugin { } } + // Ban these dependencies from all configurations + project.configurations.all { + // guava-jdk5 brings in classes which conflict with guava + exclude group: "com.google.guava", module: "guava-jdk5" + // Ban the usage of the JDK tools as a library as this is system dependent + exclude group: "jdk.tools", module: "jdk.tools" + // protobuf-lite duplicates classes which conflict with protobuf-java + exclude group: "com.google.protobuf", module: "protobuf-lite" + // Exclude these test dependencies because they bundle other common + // test libraries classes causing version conflicts. Users should rely + // on using the yyy-core package instead of the yyy-all package. + exclude group: "org.hamcrest", module: "hamcrest-all" + } + + // Force usage of the libraries defined within our common set found in the root + // build.gradle instead of using Gradles default dependency resolution mechanism + // which chooses the latest version available. + // + // TODO: Figure out whether we should force all dependency conflict resolution + // to occur in the "shadow" and "shadowTest" configurations. + project.configurations.all { config -> + // When running beam_Dependency_Check, resolutionStrategy should not be used; otherwise + // gradle-versions-plugin does not report the latest versions of the dependencies. + def startTasks = project.gradle.startParameter.taskNames + def inDependencyUpdates = 'dependencyUpdates' in startTasks || 'runBeamDependencyCheck' in startTasks + + // The "errorprone" configuration controls the classpath used by errorprone static analysis, which + // has different dependencies than our project. + if (config.getName() != "errorprone" && !inDependencyUpdates) { + config.resolutionStrategy { + // Filtering versionless coordinates that depend on BOM. Beam project needs to set the + // versions for only handful libraries when building the project (BEAM-9542). + def librariesWithVersion = project.library.java.values().findAll { it.split(':').size() > 2 } + force librariesWithVersion + + // hamcrest-core and hamcrest-library have been superseded by hamcrest. + // We force their versions here to ensure that any resolved version provides + // the same classes as hamcrest. + force "org.hamcrest:hamcrest-core:$hamcrest_version" + force "org.hamcrest:hamcrest-library:$hamcrest_version" + } + } + } + def jacocoExcludes = [ '**/org/apache/beam/gradle/**', '**/org/apache/beam/model/**', @@ -1787,50 +1831,6 @@ class BeamModulePlugin implements Plugin { } } } - - // Ban these dependencies from all configurations - project.configurations.all { - // guava-jdk5 brings in classes which conflict with guava - exclude group: "com.google.guava", module: "guava-jdk5" - // Ban the usage of the JDK tools as a library as this is system dependent - exclude group: "jdk.tools", module: "jdk.tools" - // protobuf-lite duplicates classes which conflict with protobuf-java - exclude group: "com.google.protobuf", module: "protobuf-lite" - // Exclude these test dependencies because they bundle other common - // test libraries classes causing version conflicts. Users should rely - // on using the yyy-core package instead of the yyy-all package. - exclude group: "org.hamcrest", module: "hamcrest-all" - } - - // Force usage of the libraries defined within our common set found in the root - // build.gradle instead of using Gradles default dependency resolution mechanism - // which chooses the latest version available. - // - // TODO: Figure out whether we should force all dependency conflict resolution - // to occur in the "shadow" and "shadowTest" configurations. - project.configurations.all { config -> - // When running beam_Dependency_Check, resolutionStrategy should not be used; otherwise - // gradle-versions-plugin does not report the latest versions of the dependencies. - def startTasks = project.gradle.startParameter.taskNames - def inDependencyUpdates = 'dependencyUpdates' in startTasks || 'runBeamDependencyCheck' in startTasks - - // The "errorprone" configuration controls the classpath used by errorprone static analysis, which - // has different dependencies than our project. - if (config.getName() != "errorprone" && !inDependencyUpdates) { - config.resolutionStrategy { - // Filtering versionless coordinates that depend on BOM. Beam project needs to set the - // versions for only handful libraries when building the project (BEAM-9542). - def librariesWithVersion = project.library.java.values().findAll { it.split(':').size() > 2 } - force librariesWithVersion - - // hamcrest-core and hamcrest-library have been superseded by hamcrest. - // We force their versions here to ensure that any resolved version provides - // the same classes as hamcrest. - force "org.hamcrest:hamcrest-core:$hamcrest_version" - force "org.hamcrest:hamcrest-library:$hamcrest_version" - } - } - } } def cleanUpTask = project.tasks.register('cleanUp') { dependsOn ':runners:google-cloud-dataflow-java:cleanUpDockerJavaImages' From 6ceb57fe90d8caeab8a9cb75e958816948d94a1b Mon Sep 17 00:00:00 2001 From: Andrew Pilloud Date: Thu, 17 Nov 2022 19:16:59 -0800 Subject: [PATCH 264/269] Exclude :sdks:java:core from harness jar --- CHANGES.md | 2 ++ sdks/java/harness/build.gradle | 34 ++++++++++++++++++++++------------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index db9f37eeec7f..65104de7db8f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -81,6 +81,8 @@ dependency of the Java SDK Harness. Some users of a portable runner (such as Dataflow Runner v2) may have an undeclared dependency on this package (for example using GCS with TextIO) and will now need to declare the dependency. +* `beam-sdks-java-core` is no longer a dependency of the Java SDK Harness. Users of a portable + runner (such as Dataflow Runner v2) will need to provide this package and its dependencies. ## Deprecations diff --git a/sdks/java/harness/build.gradle b/sdks/java/harness/build.gradle index 74c5e897edc7..1663c38442b5 100644 --- a/sdks/java/harness/build.gradle +++ b/sdks/java/harness/build.gradle @@ -18,6 +18,22 @@ plugins { id 'org.apache.beam.module' } +configurations { + provided +} + +dependencies { + // provided dependencies are declared early so they can be resolved in shadowClosure + + // :sdks:java:core and transitive dependencies + provided project(path: ":model:pipeline", configuration: "shadow") + provided project(path: ":sdks:java:core", configuration: "shadow") + provided library.java.joda_time + provided library.java.slf4j_api + provided library.java.vendored_grpc_1_48_1 + provided library.java.vendored_guava_26_0_jre +} + applyJavaNature( classesTriggerCheckerBugs: [ 'AssignWindowsRunner': 'https://github.com/typetools/checker-framework/issues/3794', @@ -27,13 +43,13 @@ applyJavaNature( validateShadowJar: false, testShadowJar: true, shadowClosure: - // Create an uber jar without repackaging for the SDK harness - // TODO: We have been releasing this in the past, consider not - // releasing it since its typically bad practice to release 'all' - // jars. { dependencies { - exclude(dependency(".*:avro:.*")) + // Exclude provided and transient dependencies + project.configurations.provided.getResolvedConfiguration() + .getLenientConfiguration().getAllModuleDependencies().each { + exclude(dependency(it.getModuleGroup() + ":" + it.getModuleName() + ":.*")) + } } }, ) @@ -43,19 +59,13 @@ ext.summary = "This contains the SDK Fn Harness for Beam Java" dependencies { implementation project(path: ":model:fn-execution", configuration: "shadow") - implementation project(path: ":model:pipeline", configuration: "shadow") - implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":runners:core-construction-java") implementation project(":runners:core-java") implementation project(":sdks:java:fn-execution") implementation library.java.jamm - implementation library.java.joda_time - implementation library.java.slf4j_api - implementation library.java.vendored_grpc_1_48_1 - shadow library.java.vendored_guava_26_0_jre testImplementation library.java.junit testImplementation library.java.mockito_core - testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") + shadowTestRuntimeClasspath project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation project(path: ":sdks:java:fn-execution", configuration: "testRuntimeMigration") shadowTestRuntimeClasspath library.java.slf4j_jdk14 } From 1488804f8d0fd9c56b4e7cead65ceb1a645eb18c Mon Sep 17 00:00:00 2001 From: Andrew Pilloud Date: Fri, 18 Nov 2022 11:14:46 -0800 Subject: [PATCH 265/269] Enable shadowJar validation for sdks/java/harness --- sdks/java/harness/build.gradle | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/sdks/java/harness/build.gradle b/sdks/java/harness/build.gradle index 1663c38442b5..5f7516c7772b 100644 --- a/sdks/java/harness/build.gradle +++ b/sdks/java/harness/build.gradle @@ -40,8 +40,21 @@ applyJavaNature( 'WindowMergingFnRunner': 'https://github.com/typetools/checker-framework/issues/3794', ], automaticModuleName: 'org.apache.beam.fn.harness', - validateShadowJar: false, testShadowJar: true, + shadowJarValidationExcludes: [ + "junit/**", + "io/github/classgraph/**", + "nonapi/io/github/classgraph/**", + "org/apache/beam/fn/harness/**", + "org/apache/beam/model/fnexecution/**", + "org/apache/beam/runners/core/**", + "org/apache/beam/runners/core/construction/**", + "org/apache/beam/sdk/fn/**", + "org/checkerframework/**", + "org/github/jamm/**", + "org/hamcrest/**", + "org/junit/**", + ], shadowClosure: { dependencies { From 3978caebb217663b604eea1516f2862580e81df7 Mon Sep 17 00:00:00 2001 From: Andrew Pilloud Date: Fri, 18 Nov 2022 12:54:51 -0800 Subject: [PATCH 266/269] Add missing portability runner dependencies Found by build failures and :runners:portability:java:analyzeClassesDependencies --- runners/portability/java/build.gradle | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/runners/portability/java/build.gradle b/runners/portability/java/build.gradle index ed476e255b3d..6e8e42d3c9f5 100644 --- a/runners/portability/java/build.gradle +++ b/runners/portability/java/build.gradle @@ -31,12 +31,18 @@ configurations { } dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation project(path: ":model:job-management", configuration: "shadow") + implementation project(path: ":model:pipeline", configuration: "shadow") implementation project(":runners:java-fn-execution") implementation project(":runners:java-job-service") + implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(path: ":sdks:java:harness", configuration: "shadow") implementation library.java.hamcrest permitUnusedDeclared library.java.hamcrest + implementation library.java.joda_time + implementation library.java.slf4j_api + implementation library.java.vendored_grpc_1_48_1 + implementation library.java.vendored_guava_26_0_jre testImplementation project(path: ":runners:core-construction-java", configuration: "testRuntimeMigration") testImplementation library.java.hamcrest From 2ccee1783247afe39caa941b417907a6f9751add Mon Sep 17 00:00:00 2001 From: Andrew Pilloud Date: Mon, 21 Nov 2022 15:50:41 -0800 Subject: [PATCH 267/269] Exclude jamm from harness jar --- sdks/java/harness/build.gradle | 4 ++-- sdks/java/harness/jmh/build.gradle | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/java/harness/build.gradle b/sdks/java/harness/build.gradle index 5f7516c7772b..73a3c09f3e78 100644 --- a/sdks/java/harness/build.gradle +++ b/sdks/java/harness/build.gradle @@ -32,6 +32,8 @@ dependencies { provided library.java.slf4j_api provided library.java.vendored_grpc_1_48_1 provided library.java.vendored_guava_26_0_jre + + provided library.java.jamm } applyJavaNature( @@ -51,7 +53,6 @@ applyJavaNature( "org/apache/beam/runners/core/construction/**", "org/apache/beam/sdk/fn/**", "org/checkerframework/**", - "org/github/jamm/**", "org/hamcrest/**", "org/junit/**", ], @@ -75,7 +76,6 @@ dependencies { implementation project(":runners:core-construction-java") implementation project(":runners:core-java") implementation project(":sdks:java:fn-execution") - implementation library.java.jamm testImplementation library.java.junit testImplementation library.java.mockito_core shadowTestRuntimeClasspath project(path: ":sdks:java:core", configuration: "shadowTest") diff --git a/sdks/java/harness/jmh/build.gradle b/sdks/java/harness/jmh/build.gradle index 9a284312cae1..14062a379a34 100644 --- a/sdks/java/harness/jmh/build.gradle +++ b/sdks/java/harness/jmh/build.gradle @@ -41,6 +41,7 @@ dependencies { implementation library.java.joda_time runtimeOnly library.java.slf4j_jdk14 jammAgent library.java.jamm + testRuntimeOnly library.java.jamm } jmh { From 1fa52dcf9b11a1fba7e0bb2e189899e26f54058e Mon Sep 17 00:00:00 2001 From: Andrew Pilloud Date: Mon, 28 Nov 2022 11:09:20 -0800 Subject: [PATCH 268/269] Enforce GCP BOM on sdks/java/harness --- sdks/java/harness/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/java/harness/build.gradle b/sdks/java/harness/build.gradle index 73a3c09f3e78..cf9ae66f090d 100644 --- a/sdks/java/harness/build.gradle +++ b/sdks/java/harness/build.gradle @@ -24,6 +24,7 @@ configurations { dependencies { // provided dependencies are declared early so they can be resolved in shadowClosure + provided enforcedPlatform(library.java.google_cloud_platform_libraries_bom) // :sdks:java:core and transitive dependencies provided project(path: ":model:pipeline", configuration: "shadow") @@ -72,6 +73,7 @@ description = "Apache Beam :: SDKs :: Java :: Harness" ext.summary = "This contains the SDK Fn Harness for Beam Java" dependencies { + implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) implementation project(path: ":model:fn-execution", configuration: "shadow") implementation project(":runners:core-construction-java") implementation project(":runners:core-java") From 27f523fc52d90ec93dc65ea5de5cc3c206103822 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 17:09:45 +0000 Subject: [PATCH 269/269] Bump actions/setup-java from 3.6.0 to 3.9.0 Bumps [actions/setup-java](https://github.com/actions/setup-java) from 3.6.0 to 3.9.0. - [Release notes](https://github.com/actions/setup-java/releases) - [Commits](https://github.com/actions/setup-java/compare/v3.6.0...v3.9.0) --- updated-dependencies: - dependency-name: actions/setup-java dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build_playground_backend.yml | 2 +- .github/workflows/build_playground_frontend.yml | 2 +- .github/workflows/java_tests.yml | 2 +- .github/workflows/playground_deploy_backend.yml | 2 +- .github/workflows/playground_deploy_examples.yml | 2 +- .github/workflows/playground_examples_ci_reusable.yml | 2 +- .github/workflows/run_rc_validation.yml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index f9c161c934df..e986e5ba7144 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v3 - - uses: actions/setup-java@v3.6.0 + - uses: actions/setup-java@v3.9.0 with: distribution: 'zulu' java-version: '8' diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index 62f29886c288..73d6c35f8888 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v3 - - uses: actions/setup-java@v3.6.0 + - uses: actions/setup-java@v3.9.0 with: distribution: 'zulu' java-version: '8' diff --git a/.github/workflows/java_tests.yml b/.github/workflows/java_tests.yml index 1a587e7b4919..f795b04d48b6 100644 --- a/.github/workflows/java_tests.yml +++ b/.github/workflows/java_tests.yml @@ -172,7 +172,7 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} export_default_credentials: true - name: Set Java Version - uses: actions/setup-java@v3.6.0 + uses: actions/setup-java@v3.9.0 with: distribution: 'zulu' java-version: 8 diff --git a/.github/workflows/playground_deploy_backend.yml b/.github/workflows/playground_deploy_backend.yml index 4aa0fd294931..92f269f98047 100644 --- a/.github/workflows/playground_deploy_backend.yml +++ b/.github/workflows/playground_deploy_backend.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v3.9.0 with: distribution: 'zulu' java-version: '8' diff --git a/.github/workflows/playground_deploy_examples.yml b/.github/workflows/playground_deploy_examples.yml index 8e69a7cfbc18..8e1c54613d71 100644 --- a/.github/workflows/playground_deploy_examples.yml +++ b/.github/workflows/playground_deploy_examples.yml @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.8' - - uses: actions/setup-java@v3.6.0 + - uses: actions/setup-java@v3.9.0 with: distribution: 'zulu' java-version: '8' diff --git a/.github/workflows/playground_examples_ci_reusable.yml b/.github/workflows/playground_examples_ci_reusable.yml index d00ecd3b5bdc..717170f572a9 100644 --- a/.github/workflows/playground_examples_ci_reusable.yml +++ b/.github/workflows/playground_examples_ci_reusable.yml @@ -100,7 +100,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.8' - - uses: actions/setup-java@v3.6.0 + - uses: actions/setup-java@v3.9.0 with: distribution: 'zulu' java-version: '8' diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index e8d990912ffe..130bd279179e 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -120,7 +120,7 @@ jobs: - name: Setup Java JDK - uses: actions/setup-java@v3.6.0 + uses: actions/setup-java@v3.9.0 with: distribution: 'temurin' java-version: 11 @@ -187,7 +187,7 @@ jobs: uses: azure/setup-kubectl@v3 - name: Setup Java JDK - uses: actions/setup-java@v3.6.0 + uses: actions/setup-java@v3.9.0 with: distribution: 'temurin' java-version: 11