Skip to content

Commit

Permalink
Rearrange Java Example tests on Dataflow
Browse files Browse the repository at this point in the history
  • Loading branch information
Abacn committed Sep 20, 2023
1 parent 93de970 commit e864092
Show file tree
Hide file tree
Showing 9 changed files with 138 additions and 28 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ Please note that jobs with matrix need to have matrix element in the comment. Ex
| [ PostCommit Java Dataflow V1 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml) | N/A |`Run PostCommit_Java_Dataflow`| [![.github/workflows/beam_PostCommit_Java_DataflowV1.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml) |
| [ PostCommit Java Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml) | N/A |`Run PostCommit_Java_DataflowV2`| [![.github/workflows/beam_PostCommit_Java_DataflowV2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml) |
| [ PostCommit Java Examples Dataflow ARM ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml) | N/A |`Run Java_Examples_Dataflow_ARM PostCommit`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml) |
| [ PostCommit Java Examples Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml) | N/A |`Run Java examples on Dataflow`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml) |
| [ PostCommit Java Examples Dataflow Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml) | ['11','17'] |`Run Java examples on Dataflow Java (matrix_element)`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml) |
| [ PostCommit Java Examples Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml) | N/A |`Run Java Examples_Direct`| [![.github/workflows/beam_PostCommit_Java_Examples_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml) |
| [ PostCommit Java Examples Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml) | N/A |`Run Java Examples_Flink`| [![.github/workflows/beam_PostCommit_Java_Examples_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml) |
Expand Down Expand Up @@ -281,7 +282,7 @@ Please note that jobs with matrix need to have matrix element in the comment. Ex
| [ PreCommit Java Flink Versions ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml) | N/A |`Run Java_Flink_Versions PreCommit`| [![.github/workflows/beam_PreCommit_Java_Flink_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml) |
| [ PreCommit Java GCP IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml) | N/A |`Run Java_GCP_IO_Direct PreCommit`| [![.github\workflows\beam_PreCommit_Java_GCP_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml) |
| [ PreCommit Java Examples Dataflow Java11 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml) | N/A | `Run Java_Examples_Dataflow_Java11 PreCommit` | [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml) |
| [ PreCommit Java Examples Dataflow Java17 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml) | N/A | `Run Java_Examples_Dataflow_Java17 PreCommit` | [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml) |
| [ PreCommit Java Examples Dataflow Java11 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml) | N/A | `Run Java_Examples_Dataflow_Java11 PreCommit` | [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml) |
| [ PreCommit Java File-schema-transform IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml) | N/A |`Run Java_File-schema-transform_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml) |
| [ PreCommit Java Hadoop IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml) | N/A |`Run Java_Hadoop_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml) |
| [ PreCommit Java HBase IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml) | N/A |`Run Java_HBase_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml) |
Expand Down
89 changes: 89 additions & 0 deletions .github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: PostCommit Java Examples Dataflow

on:
issue_comment:
types: [created]
schedule:
- cron: '0 */6 * * *'
workflow_dispatch:

# This allows a subsequently queued workflow run to interrupt previous runs
concurrency:
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login}}'
cancel-in-progress: true

#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
permissions:
actions: write
pull-requests: write
checks: write
contents: read
deployments: read
id-token: none
issues: write
discussions: read
packages: read
pages: read
repository-projects: read
security-events: read
statuses: read

env:
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}

jobs:
beam_PostCommit_Java_Examples_Dataflow_V2:
name: ${{matrix.job_name}} (${{matrix.job_phrase}})
runs-on: [self-hosted, ubuntu-20.04, main]
timeout-minutes: 180
strategy:
matrix:
job_name: [beam_PostCommit_Java_Examples_Dataflow_V1]
job_phrase: [Run Java examples on Dataflow]
if: |
github.event_name == 'workflow_dispatch' ||
github.event_name == 'schedule' ||
github.event.comment.body == 'Run Java Examples on Dataflow'
steps:
- uses: actions/checkout@v4
- name: Setup repository
uses: ./.github/actions/setup-action
with:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- name: run PostCommit Java Examples Dataflow script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :runners:google-cloud-dataflow-java:examples:javaPostCommit
max-workers: 12
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v3
if: failure()
with:
name: JUnit Test Results
path: "**/build/reports/tests/"
- name: Publish JUnit Test Results
uses: EnricoMi/publish-unit-test-result-action@v2
if: always()
with:
files: '**/build/test-results/**/*.xml'
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ jobs:
- name: run javaExamplesDataflowPrecommit script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :javaExamplesDataflowPrecommit
gradle-command: :runners:google-cloud-dataflow-java:examples:preCommit
arguments: |
-PdisableSpotlessCheck=true \
-PdisableCheckStyle=true \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ jobs:
- name: Build and Test
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :javaExamplesDataflowPreCommit
gradle-command: :runners:google-cloud-dataflow-java:examples:preCommit
arguments: |
-PdisableSpotlessCheck=true \
-PdisableCheckStyle=true \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ builder.build {
steps {
gradle {
rootBuildScriptDir(properties.checkoutDir)
tasks 'javaExamplesDataflowPreCommit'
tasks ':runners:google-cloud-dataflow-java:examples:preCommit'
switches '-PdisableSpotlessCheck=true'
switches '-PdisableCheckStyle=true'
switches '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK11
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ builder.build {
steps {
gradle {
rootBuildScriptDir(properties.checkoutDir)
tasks 'javaExamplesDataflowPreCommit'
tasks ':runners:google-cloud-dataflow-java:examples:preCommit'
switches '-PdisableSpotlessCheck=true'
switches '-PdisableCheckStyle=true'
switches '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK17
Expand Down
1 change: 1 addition & 0 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -557,6 +557,7 @@ tasks.register("communityMetricsProber") {
tasks.register("javaExamplesDataflowPrecommit") {
dependsOn(":runners:google-cloud-dataflow-java:examples:preCommit")
dependsOn(":runners:google-cloud-dataflow-java:examples-streaming:preCommit")
dependsOn(":runners:google-cloud-dataflow-java:examplesJavaRunnerV2PreCommit")
}

tasks.register("whitespacePreCommit") {
Expand Down
25 changes: 6 additions & 19 deletions runners/google-cloud-dataflow-java/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -622,24 +622,13 @@ task googleCloudPlatformRunnerV2IntegrationTest(type: Test) {
useJUnit { }
}

task examplesJavaLegacyWorkerIntegrationTest(type: Test) {
task examplesJavaRunnerV2PreCommit(type: Test) {
group = "Verification"
dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"

systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
"--region=${dataflowRegion}",
"--tempRoot=${dataflowPostCommitTempRoot}",
"--dataflowWorkerJar=${dataflowLegacyWorkerJar}",
"--workerHarnessContainerImage=",
])
dependsOn buildAndPushDockerJavaContainer
systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptions)
include '**/WordCountIT.class'
include '**/WindowedWordCountIT.class'

// The examples/java preCommit task already covers running WordCountIT/WindowedWordCountIT so
// this postCommit integration test excludes them.
include '**/*IT.class'
exclude '**/WordCountIT.class'
exclude '**/WindowedWordCountIT.class'
maxParallelForks 4
classpath = configurations.examplesJavaIntegrationTest
testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs)
Expand All @@ -652,7 +641,7 @@ task examplesJavaRunnerV2IntegrationTest(type: Test) {

systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptions)

// The examples/java preCommit task already covers running WordCountIT/WindowedWordCountIT so
// examplesJavaRunnerV2PreCommit task already covers running WordCountIT/WindowedWordCountIT so
// this postCommit integration test excludes them.
include '**/*IT.class'
exclude '**/WordCountIT.class'
Expand Down Expand Up @@ -710,15 +699,13 @@ task postCommit {
description = "Various integration tests using the Dataflow runner."
dependsOn googleCloudPlatformLegacyWorkerIntegrationTest
dependsOn googleCloudPlatformLegacyWorkerKmsIntegrationTest
dependsOn examplesJavaLegacyWorkerIntegrationTest
dependsOn coreSDKJavaLegacyWorkerIntegrationTest
}

task postCommitRunnerV2 {
group = "Verification"
description = "Various integration tests using the Dataflow runner V2."
dependsOn googleCloudPlatformRunnerV2IntegrationTest
dependsOn examplesJavaRunnerV2IntegrationTest
dependsOn coreSDKJavaRunnerV2IntegrationTest
}

Expand Down
40 changes: 36 additions & 4 deletions runners/google-cloud-dataflow-java/examples/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,25 @@ def dataflowWorkerImpersonationServiceAccount = project.findProperty('dataflowWo
def impersonationTempRoot = project.findProperty('gcpImpersonationTempRoot') ?: 'gs://impersonation-test-bucket/tmproot'


/*
* Set common configurations for test tasks.
*
* @param dataflowWorkerJar (required) the path te dataflow worker Jar.
* @param gcsTempRoot gcs temp root
* @additionalOptions additional options
* @workerHarnessContainerImage worker harness container image
* @param runWordCount
* "only" - only run wordcount;
* "exclude" - run example without wordcount;
* "include" - run full example
*/
def commonConfig = { Map args ->
if (!args.dataflowWorkerJar) {
throw new GradleException("Dataflow integration test configuration requires dataflowWorkerJar parameter")
}

def runWordCount = args?.runWordCount ?: 'only'

def actualDataflowWorkerJar = args.dataflowWorkerJar
def actualWorkerHarnessContainerImage = args.workerHarnessContainerImage ?: ''
def actualGcsTempRoot = args.gcsTempRoot ?: gcsTempRoot
Expand All @@ -66,8 +80,16 @@ def commonConfig = { Map args ->
// return the preevaluated configuration closure
return {
testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs)
include "**/WordCountIT.class"
include "**/WindowedWordCountIT.class"
if (runWordCount == 'only') {
include "**/WordCountIT.class"
include "**/WindowedWordCountIT.class"
} else {
include "**/IT.class"
if (runWordCount == 'exclude') {
exclude "**/WordCountIT.class"
exclude "**/WindowedWordCountIT.class"
}
}
forkEvery 1
maxParallelForks 4

Expand Down Expand Up @@ -115,11 +137,21 @@ task verifyFnApiWorker(type: Test) {
}
}

task postCommitLegacyWorker(type: Test) {
dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'exclude')
}

task javaPostCommit() {
dependsOn postCommitLegacyWorker
}

task postCommitLegacyWorkerJava11(type: Test) {
dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
systemProperty "java.specification.version", "11"
with commonConfig(dataflowWorkerJar: dataflowWorkerJar)
with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only')
}

task java11PostCommit() {
Expand All @@ -130,7 +162,7 @@ task postCommitLegacyWorkerJava17(type: Test) {
dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
systemProperty "java.specification.version", "17"
with commonConfig(dataflowWorkerJar: dataflowWorkerJar)
with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only')
}

task java17PostCommit() {
Expand Down

0 comments on commit e864092

Please sign in to comment.