From 69bed237344f2e9917ca837d537c66a6c6f0d5d8 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Tue, 18 Jul 2023 17:11:45 -0400 Subject: [PATCH 1/2] Various release script fixes * Remove Release_Gradle_Build job from release verification * Fix publish docker image script to work with remotely built images * Change empty file to .json to make RAT_PreCommit happy * Set base branch to the release branch for build snapshot PR * Bump pubsub signal wait for pipeline start to 8 min in integration tests * Change message head from Warning to Level for PubsubInjector as it follows by an exit(1) --- .github/workflows/cut_release_branch.yml | 7 +-- .github/workflows/run_rc_validation.yml | 4 +- .test-infra/jenkins/README.md | 1 - .../jenkins/job_Release_Gradle_Build.groovy | 49 ------------------- .../complete/game/injector/InjectorUtils.java | 2 +- release/src/main/scripts/jenkins_jobs.txt | 1 - .../src/main/scripts/publish_docker_images.sh | 25 +++++++--- release/src/main/scripts/run_rc_validation.sh | 4 +- .../src/main/scripts/verify_release_build.sh | 4 +- .../sdk/io/gcp/healthcare/FhirIOReadIT.java | 2 +- .../beam/sdk/io/gcp/pubsub/PubsubReadIT.java | 4 +- .../sdk/io/gcp/pubsublite/ReadWriteIT.java | 4 +- .../content/en/contribute/release-guide.md | 5 +- 13 files changed, 34 insertions(+), 78 deletions(-) delete mode 100644 .test-infra/jenkins/job_Release_Gradle_Build.groovy diff --git a/.github/workflows/cut_release_branch.yml b/.github/workflows/cut_release_branch.yml index e631fd6111b6..4e104d78a445 100644 --- a/.github/workflows/cut_release_branch.yml +++ b/.github/workflows/cut_release_branch.yml @@ -147,7 +147,7 @@ jobs: fi done - cat /tmp/result | sort | uniq | grep -i -E 'precommit|postcommit|validates|vr|example|test|gradle build' | grep -v -i -E 'load|perf|website' >> release/src/main/scripts/jenkins_jobs.txt + cat /tmp/result | sort | uniq | grep -i -E 'precommit|postcommit|validates|vr|example|test' | grep -v -i -E 'load|perf|website' >> release/src/main/scripts/jenkins_jobs.txt env: JENKINS_USERNAME: ${{ github.event.inputs.JENKINS_USERNAME }} JENKINS_TOKEN: ${{ github.event.inputs.JENKINS_TOKEN }} @@ -175,6 +175,7 @@ jobs: REMOTE_NAME: remote_repo REMOTE_URL: ${{ github.server_url }}/${{ github.repository }} BRANCH_NAME: snapshot_build-${{ github.event.inputs.RELEASE_VERSION }} + RELEASE_BRANCH: release-${{ github.event.inputs.RELEASE_VERSION }} steps: - name: Install Hub run: | @@ -192,7 +193,7 @@ jobs: run: | git remote add ${REMOTE_NAME} ${REMOTE_URL} git checkout -b ${BRANCH_NAME} - touch empty_file.txt + touch empty_file.json git add -A git commit -m "Add empty file in order to create PR" git push -f ${REMOTE_NAME} @@ -200,7 +201,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - hub pull-request -F- <<<"[DO NOT MERGE]Start snapshot build for release process + hub pull-request -b apache:${RELEASE_BRANCH} -F- <<<"[DO NOT MERGE]Start snapshot build for release process Run Gradle Publish" diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index 7a98942eed1f..720150b57450 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -92,8 +92,8 @@ jobs: - name: Create Pull Request run: | git checkout -b ${{env.WORKING_BRANCH}} ${{ env.RC_TAG }} --quiet - touch empty_file.txt - git add empty_file.txt + touch empty_file.json + git add empty_file.json git commit -m "Add empty file in order to create PR" --quiet git push origin ${{env.WORKING_BRANCH}} --quiet GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t"[DO NOT MERGE] Run Python RC Validation Tests" -b "PR to run Python ReleaseCandidate Jenkins Job.") diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 2c04fec63f52..02cddfdc65c7 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -295,7 +295,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_Publish_Beam_SDK_Snapshots | [cron](https://ci-beam.apache.org/job/beam_Publish_Beam_SDK_Snapshots/)| N/A | [![Build Status](https://ci-beam.apache.org/job/beam_Publish_Beam_SDK_Snapshots/badge/icon)](https://ci-beam.apache.org/job/beam_Publish_Beam_SDK_Snapshots/) | | beam_Publish_Docker_Snapshots | [cron](https://ci-beam.apache.org/job/beam_Publish_Docker_Snapshots/)| N/A | [![Build Status](https://ci-beam.apache.org/job/beam_Publish_Docker_Snapshots/badge/icon)](https://ci-beam.apache.org/job/beam_Publish_Docker_Snapshots/) | | beam_PostRelease_Python_Candidate | [cron](https://ci-beam.apache.org/job/beam_PostRelease_Python_Candidate/)| `Run Python ReleaseCandidate` | [![Build Status](https://ci-beam.apache.org/job/beam_PostRelease_Python_Candidate/badge/icon)](https://ci-beam.apache.org/job/beam_PostRelease_Python_Candidate/) | -| beam_Release_Gradle_Build | [cron](https://ci-beam.apache.org/job/beam_Release_Gradle_Build/) | `Run Release Gradle Build` | [![Build Status](https://ci-beam.apache.org/job/beam_Release_Gradle_Build/badge/icon)](https://ci-beam.apache.org/job/beam_Release_Gradle_Build/) ### Notes: diff --git a/.test-infra/jenkins/job_Release_Gradle_Build.groovy b/.test-infra/jenkins/job_Release_Gradle_Build.groovy deleted file mode 100644 index ba3efeca85fe..000000000000 --- a/.test-infra/jenkins/job_Release_Gradle_Build.groovy +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties - -// This job runs complete cycle of Gradle build against the official release -// version. Release manager should use this job to verify a release branch -// after cut. -job('beam_Release_Gradle_Build') { - description('Verify Gradle build against the official release version.') - - // Set common parameters. - commonJobProperties - .setTopLevelMainJobProperties( - delegate, - defaultBranch='master', - defaultTimeout=300) - - // Allows triggering this build against pull requests. - commonJobProperties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Release_Build ("Run Release Gradle Build")', - 'Run Release Gradle Build') - - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks('build') - commonJobProperties.setGradleSwitches(delegate) - switches('-PisRelease') - switches('--stacktrace') - } - } -} diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java b/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java index dbefed2b7cc3..4713d15a05c2 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java @@ -47,7 +47,7 @@ public static Pubsub getClient(final HttpTransport httpTransport, final JsonFact } if (credential.getClientAuthentication() != null) { System.out.println( - "\n***Warning! You are not using service account credentials to " + "\n***Error! You are not using service account credentials to " + "authenticate.\nYou need to use service account credentials for this example," + "\nsince user-level credentials do not have enough pubsub quota,\nand so you will run " + "out of PubSub quota very quickly.\nSee " diff --git a/release/src/main/scripts/jenkins_jobs.txt b/release/src/main/scripts/jenkins_jobs.txt index 00db48bee486..c32b47f1dbf6 100644 --- a/release/src/main/scripts/jenkins_jobs.txt +++ b/release/src/main/scripts/jenkins_jobs.txt @@ -139,7 +139,6 @@ Run Python_Transforms PreCommit,beam_PreCommit_Python_Transforms_Phrase Run Python_Xlang_Gcp_Dataflow PostCommit,beam_PostCommit_Python_Xlang_Gcp_Dataflow_PR Run Python_Xlang_Gcp_Direct PostCommit,beam_PostCommit_Python_Xlang_Gcp_Direct_PR Run RAT PreCommit,beam_PreCommit_RAT_Phrase -Run Release Gradle Build,beam_Release_Gradle_Build Run SQL PostCommit,beam_PostCommit_SQL_PR Run SQL PreCommit,beam_PreCommit_SQL_Phrase Run SQL_Java11 PreCommit,beam_PreCommit_SQL_Java11_Phrase diff --git a/release/src/main/scripts/publish_docker_images.sh b/release/src/main/scripts/publish_docker_images.sh index b189c9c1bfdc..d7c1d1b9599e 100755 --- a/release/src/main/scripts/publish_docker_images.sh +++ b/release/src/main/scripts/publish_docker_images.sh @@ -37,18 +37,27 @@ echo "Which release candidate will be the source of final docker images? (ex: 1) read RC_NUM RC_VERSION="rc${RC_NUM}" -echo "================Confirming Release and RC version===========" +echo "================Pull RC Containers from DockerHub===========" +IMAGES=$(docker search ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX} --format "{{.Name}}" --limit 100) +KNOWN_IMAGES=() echo "We are using ${RC_VERSION} to push docker images for ${RELEASE}." +while read IMAGE; do + # Try pull verified RC from dockerhub. + if docker pull "${IMAGE}:${RELEASE}${RC_VERSION}" 2>/dev/null ; then + KNOWN_IMAGES+=( $IMAGE ) + fi +done < <(echo "${IMAGES}") + +echo "================Confirming Release and RC version===========" echo "Publishing the following images:" -IMAGES=$(docker images --filter "reference=apache/beam_*:${RELEASE}${RC_VERSION}" --format "{{.Repository}}") -echo "${IMAGES}" +# Sort by name for easy examination +IFS=$'\n' KNOWN_IMAGES=($(sort <<<"${KNOWN_IMAGES[*]}")) +unset IFS +printf "%s\n" ${KNOWN_IMAGES[@]} echo "Do you want to proceed? [y|N]" read confirmation if [[ $confirmation = "y" ]]; then - echo "${IMAGES}" | while read IMAGE; do - # Pull verified RC from dockerhub. - docker pull "${IMAGE}:${RELEASE}${RC_VERSION}" - + for IMAGE in "${KNOWN_IMAGES[@]}"; do # Tag with ${RELEASE} and push to dockerhub. docker tag "${IMAGE}:${RELEASE}${RC_VERSION}" "${IMAGE}:${RELEASE}" docker push "${IMAGE}:${RELEASE}" @@ -58,4 +67,4 @@ if [[ $confirmation = "y" ]]; then docker push "${IMAGE}:latest" done -fi \ No newline at end of file +fi diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh index 4ad8af16723b..7f32c2979660 100755 --- a/release/src/main/scripts/run_rc_validation.sh +++ b/release/src/main/scripts/run_rc_validation.sh @@ -271,8 +271,8 @@ echo "This task will create a PR against apache/beam, trigger a jenkins job to r echo "1. Python quickstart validations(batch & streaming)" echo "2. Python MobileGame validations(UserScore, HourlyTeamScore)" if [[ "$python_quickstart_mobile_game" = true && ! -z `which hub` ]]; then - touch empty_file.txt - git add empty_file.txt + touch empty_file.json + git add empty_file.json git commit -m "Add empty file in order to create PR" --quiet git push -f ${GITHUB_USERNAME} --quiet # Create a test PR diff --git a/release/src/main/scripts/verify_release_build.sh b/release/src/main/scripts/verify_release_build.sh index 0f9921c2bf86..214c65cc9ef6 100755 --- a/release/src/main/scripts/verify_release_build.sh +++ b/release/src/main/scripts/verify_release_build.sh @@ -125,8 +125,8 @@ hub version echo "" -echo "==================== 3 Run Gradle Release Build & PostCommit Tests on Jenkins ===================" -echo "[Current Task] Run Gradle release build and all PostCommit Tests against Release Branch on Jenkins." +echo "==================== 3 Run PostCommit Tests on Jenkins ===================" +echo "[Current Task] Run all PostCommit Tests against Release Branch on Jenkins." echo "This task will create a PR against apache/beam." echo "After PR created, you need to comment phrases listed in description in the created PR:" diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOReadIT.java index 142b26dd2cdf..a9db2fc240eb 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOReadIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOReadIT.java @@ -133,7 +133,7 @@ public void testFhirIORead() throws Exception { "waitForAnyMessage", signal.signalSuccessWhen(resources.getCoder(), anyResources -> true)); // wait for any resource - Supplier start = signal.waitForStart(Duration.standardMinutes(5)); + Supplier start = signal.waitForStart(Duration.standardMinutes(8)); pipeline.apply(signal.signalStart()); PipelineResult job = pipeline.run(); start.get(); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadIT.java index 7b370ebf7e2c..193ba5e19c40 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadIT.java @@ -51,7 +51,7 @@ public void testReadPublicData() throws Exception { messages.apply( "waitForAnyMessage", signal.signalSuccessWhen(messages.getCoder(), anyMessages -> true)); - Supplier start = signal.waitForStart(Duration.standardMinutes(5)); + Supplier start = signal.waitForStart(Duration.standardMinutes(8)); pipeline.apply(signal.signalStart()); PipelineResult job = pipeline.run(); start.get(); @@ -79,7 +79,7 @@ public void testReadPubsubMessageId() throws Exception { "isMessageIdNonNull", signal.signalSuccessWhen(messages.getCoder(), new NonEmptyMessageIdCheck())); - Supplier start = signal.waitForStart(Duration.standardMinutes(5)); + Supplier start = signal.waitForStart(Duration.standardMinutes(8)); pipeline.apply(signal.signalStart()); PipelineResult job = pipeline.run(); start.get(); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java index 89a70a642f50..fd01cf9d0068 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java @@ -318,7 +318,7 @@ public void testPubsubLiteWriteReadWithSchemaTransform() throws Exception { return Objects.requireNonNull(row.getInt64("numberInInt")).intValue(); })); ids.apply("PubsubSignalTest", signal.signalSuccessWhen(BigEndianIntegerCoder.of(), testIds())); - Supplier start = signal.waitForStart(Duration.standardMinutes(5)); + Supplier start = signal.waitForStart(Duration.standardMinutes(8)); pipeline.apply("start signal", signal.signalStart()); PipelineResult job = pipeline.run(); start.get(); @@ -363,7 +363,7 @@ public void testReadWrite() throws Exception { PCollection messages = readMessages(subscription, pipeline); PCollection ids = messages.apply(MapElements.via(extractIds())); ids.apply("PubsubSignalTest", signal.signalSuccessWhen(BigEndianIntegerCoder.of(), testIds())); - Supplier start = signal.waitForStart(Duration.standardMinutes(5)); + Supplier start = signal.waitForStart(Duration.standardMinutes(8)); pipeline.apply(signal.signalStart()); PipelineResult job = pipeline.run(); start.get(); diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md index 7bca90daefd3..3b0a0cb2c9a4 100644 --- a/website/www/site/content/en/contribute/release-guide.md +++ b/website/www/site/content/en/contribute/release-guide.md @@ -318,7 +318,7 @@ There are 2 ways to perform this verification, either running automation script( ``` cd beam/release/src/main/scripts && ./verify_release_build.sh ``` - 1. Trigger `beam_Release_Gradle_Build` and all Jenkins PostCommit jobs from the PR created by the previous step. + 1. Trigger all Jenkins PostCommit jobs from the PR created by the previous step. You can run [mass_comment.py](https://github.com/apache/beam/blob/master/release/src/main/scripts/mass_comment.py) to do that. Or manually add one trigger phrase per PR comment. See [jenkins_jobs.txt](https://github.com/apache/beam/blob/master/release/src/main/scripts/jenkins_jobs.txt) @@ -328,9 +328,6 @@ There are 2 ways to perform this verification, either running automation script( 1. Installs ```hub``` with your agreement and setup local git repo; 1. Create a test PR against release branch; -The [`beam_Release_Gradle_Build`](https://ci-beam.apache.org/job/beam_Release_Gradle_Build/) Jenkins job runs `./gradlew build -PisRelease`. -This only verifies that everything builds with unit tests passing. - #### Verify the build succeeds * Tasks you need to do manually to __verify the build succeed__: From 759fbf7681836bb2c925c124df1c224ca2e200d9 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Tue, 18 Jul 2023 18:08:02 -0400 Subject: [PATCH 2/2] Fix Pylint --- sdks/python/apache_beam/runners/worker/sdk_worker_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py index ccfc27d0101c..8570c5a7722c 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py @@ -23,7 +23,6 @@ import logging import unittest from collections import namedtuple -from typing import Any import grpc import hamcrest as hc