Merge branch 'apache:master' into add-py-typed

.asf.yaml

@@ -49,6 +49,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.50.0: {}
     release-2.49.0: {}
     release-2.48.0: {}
     release-2.47.0: {}

.github/REVIEWERS.yml

@@ -45,7 +45,6 @@ labels:
     reviewers:
       - chamikaramj
       - johnjcasey
-      - pabloem
       - Abacn
       - ahmedabu98
       - bvolpato

.github/actions/setup-action/action.yml

@@ -39,15 +39,23 @@ runs:
         echo "The comment ${{ github.event.comment.body }} does not match the phrase for this instance:  ${{ inputs.comment_phrase }}. Exiting."
         exit 1
     - name: Check out repository code if pull request commit
-      uses: actions/checkout@v3
+      shell: bash
       if: ${{ github.event_name == 'pull_request_target' }}
-      with:
-        ref: refs/pull/${{ github.event.number }}/merge
+      run: |
+          # GitHub will automatically generate a merge commit when there are no merge conflicts.
+          # We first try to check that out, and fall back to checking out the tip of the pull request branch.
+          git fetch --depth=1 origin +refs/pull/${{ github.event.number }}/merge:refs/remotes/pull/${{ github.event.number }}/merge || \
+          git fetch --depth=1 origin +refs/pull/${{ github.event.number }}/head:refs/remotes/pull/${{ github.event.number }}/head
+          git checkout pull/${{ github.event.number }}/merge || git checkout pull/${{ github.event.number }}/head
     - name: Check out repository code if comment phrase
-      uses: actions/checkout@v3
+      shell: bash
       if: ${{ github.event.comment.body == inputs.comment_phrase }}
-      with:
-        ref: refs/pull/${{ github.event.issue.number }}/merge
+      run: |
+          # GitHub will automatically generate a merge commit when there are no merge conflicts.
+          # We first try to check that out, and fall back to checking out the tip of the pull request branch.
+          git fetch --depth=1 origin +refs/pull/${{ github.event.issue.number }}/merge:refs/remotes/pull/${{ github.event.issue.number }}/merge || \
+          git fetch --depth=1 origin +refs/pull/${{ github.event.issue.number }}/head:refs/remotes/pull/${{ github.event.issue.number }}/head
+          git checkout pull/${{ github.event.issue.number }}/merge || git checkout pull/${{ github.event.issue.number }}/head
     - name: Rerun if comment phrase
       if: ${{ github.event.comment.body == inputs.comment_phrase }}
       uses: ./.github/actions/rerun-job-action
@@ -57,3 +65,8 @@ runs:
         github_repository: ${{ github.repository }}     
         github_current_run_id: ${{ github.run_id }}
         pull_request_url: ${{ github.event.issue.pull_request.url }}
+    ## Used for jobs that spawn docker containers and need to mount gcloud config directory
+    - name: expose gcloud path
+      shell: bash
+      run: |
+        echo KUBELET_GCLOUD_CONFIG_PATH=/var/lib/kubelet/pods/$POD_UID/volumes/kubernetes.io~empty-dir/gcloud >> $GITHUB_ENV

.github/codecov.yml

@@ -65,6 +65,7 @@ ignore:
   - "**/*_microbenchmark.py"
   - "sdks/go/pkg/beam/register/register.go"
   - "sdks/python/apache_beam/testing/benchmarks/nexmark/**"
+  - "sdks/python/apache_beam/examples/**"
 
 # See https://docs.codecov.com/docs/flags for options.
 flag_management:

.github/gh-actions-self-hosted-runners/arc/environments/beam.env

@@ -31,12 +31,12 @@ deploy_webhook = "true"
 runner_group = "beam"
 main_runner = {
     name = "main-runner"
-    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:8b42b8dcea17eccff9c1ce4ae6e208daacfb9e31"
+    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:60d397ecfbd2b10a1929615c70d500eb71a2c053"
     machine_type = "e2-standard-16"
     min_node_count = "1"
-    max_node_count = "16"
+    max_node_count = "24"
     min_replicas = "1"
-    max_replicas = "128"
+    max_replicas = "200"
     webhook_scaling = true
     disk_size_gb = 200
     requests = {
@@ -47,7 +47,7 @@ main_runner = {
 additional_runner_pools = [{
     name = "small-runner"
     machine_type = "e2-standard-2"
-    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:8b42b8dcea17eccff9c1ce4ae6e208daacfb9e31"
+    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:60d397ecfbd2b10a1929615c70d500eb71a2c053"
     min_node_count = "1"
     max_node_count = "10"
     min_replicas = "1"
@@ -60,5 +60,22 @@ additional_runner_pools = [{
     labels = ["self-hosted", "ubuntu-20.04", "small"]
     enable_selector = true
     enable_taint = true
+},
+{
+    name = "highmem-runner"
+    machine_type = "c3-highmem-8"
+    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:60d397ecfbd2b10a1929615c70d500eb71a2c053"
+    min_node_count = "1"
+    max_node_count = "10"
+    min_replicas = "1"
+    max_replicas = "10"
+    webhook_scaling = "true"
+    requests = {
+        cpu = "7.5"
+        memory = "5Gi"
+    }
+    labels = ["self-hosted", "ubuntu-20.04", "highmem"]
+    enable_selector = true
+    enable_taint = true
 }]
 #state_bucket_name = "beam-arc-state"

.github/gh-actions-self-hosted-runners/arc/images/Dockerfile

@@ -59,11 +59,24 @@ RUN curl -OL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-c
     rm google-cloud-sdk-367.0.0-linux-x86_64.tar.gz && \
     mv google-cloud-sdk /usr/local/google-cloud-sdk && \
     /usr/local/google-cloud-sdk/install.sh --quiet && \
-    /usr/local/google-cloud-sdk/bin/gcloud components install kubectl && \
+    /usr/local/google-cloud-sdk/bin/gcloud components install gke-gcloud-auth-plugin && \
     #revert permission
     chown -R runner:runner /home/runner/.config
+ENV USE_GKE_GCLOUD_AUTH_PLUGIN=True
 ENV PATH="${PATH}:/usr/local/google-cloud-sdk/bin"
+#Install Kubectl
+RUN curl -OL https://dl.k8s.io/release/v1.28.1/bin/linux/amd64/kubectl && \
+    chmod +x ./kubectl && \
+    mv ./kubectl /usr/local/bin/kubectl
+#Install Apache Maven
+RUN curl -OL https://dlcdn.apache.org/maven/maven-3/3.9.4/binaries/apache-maven-3.9.4-bin.tar.gz && \
+    tar -xvf apache-maven-3.9.4-bin.tar.gz && \
+    rm apache-maven-3.9.4-bin.tar.gz && \
+    mv apache-maven-3.9.4 /usr/local/maven
+ENV PATH="${PATH}:/usr/local/maven/bin"
+ENV MAVEN_HOME="/usr/local/maven"
 
 # Needed to transfer path addtitions to runner environment
 RUN echo PATH=$PATH >> /runnertmp/.env
+RUN echo USE_GKE_GCLOUD_AUTH_PLUGIN=$USE_GKE_GCLOUD_AUTH_PLUGIN >> /runnertmp/.env
 USER runner