From e7a6405800a83dd16437b8b1b372e020e010a042 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Fri, 20 Oct 2023 11:39:04 -0400 Subject: [PATCH] Setup Java 21 container (#28918) * Setup Java 21 container (#28833) * Add Java21 container * fix spotless * Update website * Fix jamm * align configs in recent change * incremental fixes * add more comments for when jpms test enabled --- ...a_CoGBK_Dataflow_V2_Batch_JavaVersions.yml | 8 ++-- ...GBK_Dataflow_V2_Streaming_JavaVersions.yml | 8 ++-- ...ests_Java_GBK_Dataflow_V2_Batch_Java11.yml | 14 +++---- ...ests_Java_GBK_Dataflow_V2_Batch_Java17.yml | 14 +++---- ..._Java_GBK_Dataflow_V2_Streaming_Java11.yml | 14 +++---- ..._Java_GBK_Dataflow_V2_Streaming_Java17.yml | 14 +++---- ...a_ParDo_Dataflow_V2_Batch_JavaVersions.yml | 8 ++-- ...rDo_Dataflow_V2_Streaming_JavaVersions.yml | 8 ++-- ..._PostCommit_Java_Examples_Dataflow_ARM.yml | 2 +- ...tCommit_Java_Examples_Dataflow_V2_Java.yml | 2 +- ...m_PostCommit_Java_Jpms_Dataflow_Java17.yml | 2 +- ...eam_PostCommit_Java_Jpms_Direct_Java17.yml | 2 +- ...stCommit_Java_Nexmark_Dataflow_V2_Java.yml | 2 +- ...eam_PostCommit_TransformService_Direct.yml | 2 +- ...reCommit_Java_Examples_Dataflow_Java11.yml | 2 +- ...reCommit_Java_Examples_Dataflow_Java17.yml | 2 +- .../workflows/beam_PreCommit_SQL_Java11.yml | 2 +- .../workflows/beam_PreCommit_SQL_Java17.yml | 2 +- .test-infra/jenkins/JavaTestProperties.groovy | 7 +++- .test-infra/jenkins/NexmarkBuilder.groovy | 4 +- ..._LoadTests_CoGBK_Dataflow_V2_Java11.groovy | 2 +- ..._LoadTests_CoGBK_Dataflow_V2_Java17.groovy | 2 +- ...ob_LoadTests_GBK_Dataflow_V2_Java11.groovy | 2 +- ...ob_LoadTests_GBK_Dataflow_V2_Java17.groovy | 2 +- ..._LoadTests_ParDo_Dataflow_V2_Java11.groovy | 2 +- ..._LoadTests_ParDo_Dataflow_V2_Java17.groovy | 2 +- ...it_Java_Examples_Dataflow_V2_Java11.groovy | 2 +- ...it_Java_Examples_Dataflow_V2_Java17.groovy | 2 +- ...ostCommit_Java_Jpms_Dataflow_Java17.groovy | 2 +- ..._PostCommit_Java_Jpms_Direct_Java17.groovy | 2 +- ..._PostCommit_TransformService_Direct.groovy | 2 +- build.gradle.kts | 14 +++---- .../beam/gradle/BeamModulePlugin.groovy | 37 ++++++++++++++++++- .../core/construction/Environments.java | 6 ++- .../core/construction/EnvironmentsTest.java | 6 ++- .../arm/build.gradle | 6 +-- .../google-cloud-dataflow-java/build.gradle | 6 +-- .../beam/runners/dataflow/DataflowRunner.java | 7 ++-- runners/spark/spark_runner.gradle | 5 ++- sdks/java/container/Dockerfile | 3 +- sdks/java/container/agent/build.gradle | 14 ++++++- sdks/java/container/common.gradle | 28 +++++++++++--- sdks/java/container/java21/build.gradle | 30 +++++++++++++++ sdks/java/container/java21/option-jamm.json | 12 ++++++ sdks/java/testing/jpms-tests/build.gradle | 25 +++++++------ sdks/java/testing/test-utils/build.gradle | 29 +++++---------- .../jvmverification/JvmVerification.java | 16 +++++++- settings.gradle.kts | 1 + .../www/site/content/en/roadmap/java-sdk.md | 6 +-- 49 files changed, 255 insertions(+), 137 deletions(-) create mode 100644 sdks/java/container/java21/build.gradle create mode 100644 sdks/java/container/java21/option-jamm.json diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml index fea2eb18ff14..327e007d5437 100644 --- a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml @@ -95,7 +95,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ @@ -105,7 +105,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ @@ -115,7 +115,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ @@ -125,7 +125,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml index b81ea53ab5fd..0abe99ce83d2 100644 --- a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml @@ -95,7 +95,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ @@ -105,7 +105,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ @@ -115,7 +115,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ @@ -125,7 +125,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml index 3a8d47ccb642..39490d59fc22 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml @@ -95,7 +95,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_1 }}' \ - name: run Load test 2GB of 100B records @@ -106,7 +106,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_2 }}' \ - name: run Load test 2GB of 100kB records @@ -117,7 +117,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_3 }}' \ - name: run Load test fanout 4 times with 2GB 10-byte records total @@ -128,7 +128,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_4 }}' \ - name: run Load test fanout 8 times with 2GB 10-byte records total @@ -139,7 +139,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_5 }}' \ - name: run Load test reiterate 4 times 10kB values @@ -150,7 +150,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_6 }}' \ - name: run Load test reiterate 4 times 2MB values @@ -161,6 +161,6 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml index 414b2e0d660c..bb4e039ec8c2 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml @@ -97,7 +97,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_1 }}' \ - name: run Load test 2GB of 100B records @@ -108,7 +108,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_2 }}' \ - name: run Load test 2GB of 100kB records @@ -119,7 +119,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_3 }}' \ - name: run Load test fanout 4 times with 2GB 10-byte records total @@ -130,7 +130,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_4 }}' \ - name: run Load test fanout 8 times with 2GB 10-byte records total @@ -141,7 +141,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_5 }}' \ - name: run Load test reiterate 4 times 10kB values @@ -152,7 +152,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_6 }}' \ - name: run Load test reiterate 4 times 2MB values @@ -163,6 +163,6 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml index bfd8dfe09d9c..60cf635457de 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml @@ -95,7 +95,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_1 }}' \ - name: run Load test 2GB of 100B records @@ -106,7 +106,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_2 }}' \ - name: run Load test 2GB of 100kB records @@ -117,7 +117,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_3 }}' \ - name: run Load test fanout 4 times with 2GB 10-byte records total @@ -128,7 +128,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_4 }}' \ - name: run Load test fanout 8 times with 2GB 10-byte records total @@ -139,7 +139,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_5 }}' \ - name: run Load test reiterate 4 times 10kB values @@ -150,7 +150,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_6 }}' \ - name: run Load test reiterate 4 times 2MB values @@ -161,6 +161,6 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml index 22df3fe88d26..24d218c152ed 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml @@ -97,7 +97,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_1 }}' \ - name: run Load test 2GB of 100B records @@ -108,7 +108,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_2 }}' \ - name: run Load test 2GB of 100kB records @@ -119,7 +119,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_3 }}' \ - name: run Load test fanout 4 times with 2GB 10-byte records total @@ -130,7 +130,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_4 }}' \ - name: run Load test fanout 8 times with 2GB 10-byte records total @@ -141,7 +141,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_5 }}' \ - name: run Load test reiterate 4 times 10kB values @@ -152,7 +152,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_6 }}' \ - name: run Load test reiterate 4 times 2MB values @@ -163,6 +163,6 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -Pjava17Home=$JAVA_HOME_17_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml index a283310ff1d0..66b5b5a8837f 100644 --- a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml @@ -98,7 +98,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_1 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_1' \ - name: run ParDo Dataflow V2 Batch Java Load Test 2 (200 times) @@ -109,7 +109,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_2 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_2' \ - name: run ParDo Dataflow V2 Batch Java Load Test 3 (10 counters) @@ -120,7 +120,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_3 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_3' \ - name: run ParDo Dataflow V2 Batch Java Load Test 4 (100 counters) @@ -131,6 +131,6 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_4 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_4' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml index 45ea975d1580..bd8987d4ee0d 100644 --- a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml @@ -98,7 +98,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_1 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_1' \ - name: run ParDo Dataflow V2 Streaming Java Load Test 2 (200 times) @@ -109,7 +109,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_2 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_2' \ - name: run ParDo Dataflow V2 Streaming Java Load Test 3 (10 counters) @@ -120,7 +120,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_3 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_3' \ - name: run ParDo Dataflow V2 Streaming Java Load Test 4 (100 counters) @@ -131,6 +131,6 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_4 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_4' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml index c401c27922fb..09d90d5df1a8 100644 --- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml @@ -111,7 +111,7 @@ jobs: gradle-command: :runners:google-cloud-dataflow-java:arm:examplesJavaRunnerV2IntegrationTestARM max-workers: 12 arguments: | - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ -Pcontainer-architecture-list=arm64,amd64 \ -Ppush-containers \ diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml index 13506595fe1a..411cc65944fb 100644 --- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml @@ -86,7 +86,7 @@ jobs: -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ -PskipCheckerFramework \ - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v3 diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml index 69b5fc23e07a..ff27bae8c527 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml @@ -80,7 +80,7 @@ jobs: gradle-command: :sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest arguments: -PskipCheckerFramework - -PcompileAndRunTestsWithJava17 + -PtestJavaVersion=17 -Pjava17Home=$JAVA_HOME_17_X64 - name: Archive JUnit Test Results uses: actions/upload-artifact@v3 diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml index 946e34675385..748a74d2ec15 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml @@ -80,7 +80,7 @@ jobs: gradle-command: :sdks:java:testing:jpms-tests:directRunnerIntegrationTest arguments: -PskipCheckerFramework - -PcompileAndRunTestsWithJava17 + -PtestJavaVersion=17 -Pjava17Home=$JAVA_HOME_17_X64 - name: Archive JUnit Test Results uses: actions/upload-artifact@v3 diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml index 2bd36200b217..076c030d91ea 100644 --- a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml +++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml @@ -107,7 +107,7 @@ jobs: with: gradle-command: :sdks:java:testing:nexmark:run arguments: | - -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ -Pnexmark.runner.version=V2 \ -Pnexmark.runner=:runners:google-cloud-dataflow-java \ diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml index 916cd3b34e75..4c9bb3aa9302 100644 --- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml +++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml @@ -84,7 +84,7 @@ jobs: with: gradle-command: :sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava arguments: | - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ -PuseWheelDistribution \ -PpythonVersion=${{ matrix.python_version }} \ diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml index 8b584bc64122..b12590c01603 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml @@ -116,7 +116,7 @@ jobs: -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ -PskipCheckerFramework \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -Pjava11Home=$JAVA_HOME_11_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v3 diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml index e4e4b62a2e8a..d4a55aa3a824 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml @@ -116,7 +116,7 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -PskipCheckerFramework \ -Pjava17Home=$JAVA_HOME_17_X64 \ max-workers: 12 diff --git a/.github/workflows/beam_PreCommit_SQL_Java11.yml b/.github/workflows/beam_PreCommit_SQL_Java11.yml index c48ad0dd8063..4acbecc8d0fc 100644 --- a/.github/workflows/beam_PreCommit_SQL_Java11.yml +++ b/.github/workflows/beam_PreCommit_SQL_Java11.yml @@ -103,7 +103,7 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - -PcompileAndRunTestsWithJava11 \ + -PtestJavaVersion=11 \ -PskipCheckerFramework \ -Pjava11Home=$JAVA_HOME_11_X64 \ - name: Archive JUnit Test Results diff --git a/.github/workflows/beam_PreCommit_SQL_Java17.yml b/.github/workflows/beam_PreCommit_SQL_Java17.yml index 1ea901947a98..a5899fff4837 100644 --- a/.github/workflows/beam_PreCommit_SQL_Java17.yml +++ b/.github/workflows/beam_PreCommit_SQL_Java17.yml @@ -101,7 +101,7 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - -PcompileAndRunTestsWithJava17 \ + -PtestJavaVersion=17 \ -PskipCheckerFramework \ -Pjava17Home=$JAVA_HOME_17_X64 \ - name: Archive JUnit Test Results diff --git a/.test-infra/jenkins/JavaTestProperties.groovy b/.test-infra/jenkins/JavaTestProperties.groovy index ce7446a6e71d..5403cee5cf9a 100644 --- a/.test-infra/jenkins/JavaTestProperties.groovy +++ b/.test-infra/jenkins/JavaTestProperties.groovy @@ -17,5 +17,10 @@ */ class JavaTestProperties { - final static List SUPPORTED_CONTAINER_TASKS = ['java8', 'java11', 'java17'] + final static List SUPPORTED_CONTAINER_TASKS = [ + 'java8', + 'java11', + 'java17', + 'java21' + ] } diff --git a/.test-infra/jenkins/NexmarkBuilder.groovy b/.test-infra/jenkins/NexmarkBuilder.groovy index 044b0cbb9561..69fa3dcc4277 100644 --- a/.test-infra/jenkins/NexmarkBuilder.groovy +++ b/.test-infra/jenkins/NexmarkBuilder.groovy @@ -145,7 +145,7 @@ class NexmarkBuilder { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches("-PcompileAndRunTestsWithJava11") + switches("-PtestJavaVersion=11") switches("-Pjava11Home=${commonJobProperties.JAVA_11_HOME}") switches("-Pnexmark.runner=${runner.getDependencyBySDK(sdk)}") switches("-Pnexmark.args=\"${parseOptions(options)}\"") @@ -168,7 +168,7 @@ class NexmarkBuilder { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches("-PcompileAndRunTestsWithJava17") + switches("-PtestJavaVersion=17") switches("-Pjava17Home=${commonJobProperties.JAVA_17_HOME}") switches("-Pnexmark.runner=${runner.getDependencyBySDK(sdk)}") switches("-Pnexmark.args=\"${parseOptions(options)}\"") diff --git a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy index fc7f39d28a0d..55501db4429d 100644 --- a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy @@ -188,7 +188,7 @@ def loadTestConfigurations = { mode, isStreaming -> def final JOB_SPECIFIC_SWITCHES = [ '-Prunner.version="V2"', - '-PcompileAndRunTestsWithJava11', + '-PtestJavaVersion=11', "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}" ] diff --git a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy index ca8c6689ad0f..8fb09fd07448 100644 --- a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy @@ -188,7 +188,7 @@ def loadTestConfigurations = { mode, isStreaming -> def final JOB_SPECIFIC_SWITCHES = [ '-Prunner.version="V2"', - '-PcompileAndRunTestsWithJava17', + '-PtestJavaVersion=17', "-Pjava17Home=${commonJobProperties.JAVA_17_HOME}" ] diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy index cc2d5d2e5554..2191e448fade 100644 --- a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy @@ -253,7 +253,7 @@ def loadTestConfigurations = { mode, isStreaming -> def final JOB_SPECIFIC_SWITCHES = [ '-Prunner.version="V2"', - '-PcompileAndRunTestsWithJava11', + '-PtestJavaVersion=11', "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}" ] diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy index 7405f9154b83..2520f68f0178 100644 --- a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy @@ -253,7 +253,7 @@ def loadTestConfigurations = { mode, isStreaming -> def final JOB_SPECIFIC_SWITCHES = [ '-Prunner.version="V2"', - '-PcompileAndRunTestsWithJava17', + '-PtestJavaVersion=17', "-Pjava17Home=${commonJobProperties.JAVA_17_HOME}" ] diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy index a4535d52e6cf..b7154e840e6a 100644 --- a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy @@ -160,7 +160,7 @@ def commonLoadTestConfig = { jobType, isStreaming -> def final JOB_SPECIFIC_SWITCHES = [ '-Prunner.version="V2"', - '-PcompileAndRunTestsWithJava11', + '-PtestJavaVersion=11', "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}" ] diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy index f0f2179ebb3b..df6c66e02aed 100644 --- a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy @@ -160,7 +160,7 @@ def commonLoadTestConfig = { jobType, isStreaming -> def final JOB_SPECIFIC_SWITCHES = [ '-Prunner.version="V2"', - '-PcompileAndRunTestsWithJava17', + '-PtestJavaVersion=17', "-Pjava17Home=${commonJobProperties.JAVA_17_HOME}" ] diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy index 6687ae0e6f8a..6229f7c48a72 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy @@ -43,7 +43,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) switches '-PdisableSpotlessCheck=true' switches '-PdisableCheckStyle=true' - switches '-PcompileAndRunTestsWithJava11' + switches '-PtestJavaVersion=11' switches '-PskipCheckerFramework' switches "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}" } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy index b275fe9276d9..7e52a7e09789 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy @@ -43,7 +43,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) switches '-PdisableSpotlessCheck=true' switches '-PdisableCheckStyle=true' - switches '-PcompileAndRunTestsWithJava17' + switches '-PtestJavaVersion=17' switches '-PskipCheckerFramework' switches "-Pjava17Home=${commonJobProperties.JAVA_17_HOME}" } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Dataflow_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Dataflow_Java17.groovy index 4e26c164319e..f518985ca7a8 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Dataflow_Java17.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Dataflow_Java17.groovy @@ -42,7 +42,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Jpms_Dataflow_Java17', tasks(':sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest') commonJobProperties.setGradleSwitches(delegate) switches("-PskipCheckerFramework") - switches("-PcompileAndRunTestsWithJava17") + switches("-PtestJavaVersion=17") switches("-Pjava17Home=${commonJobProperties.JAVA_17_HOME}") // Specify maven home on Jenkins, needed by Maven archetype integration tests. switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') diff --git a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Direct_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Direct_Java17.groovy index f31373ecaada..04c31389ecbe 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Direct_Java17.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Direct_Java17.groovy @@ -42,7 +42,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Jpms_Direct_Java17', 'R tasks(':sdks:java:testing:jpms-tests:directRunnerIntegrationTest') commonJobProperties.setGradleSwitches(delegate) switches("-PskipCheckerFramework") - switches("-PcompileAndRunTestsWithJava17") + switches("-PtestJavaVersion=17") switches("-Pjava17Home=${commonJobProperties.JAVA_17_HOME}") // Specify maven home on Jenkins, needed by Maven archetype integration tests. switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') diff --git a/.test-infra/jenkins/job_PostCommit_TransformService_Direct.groovy b/.test-infra/jenkins/job_PostCommit_TransformService_Direct.groovy index 0d7f58e71706..03d29069a52c 100644 --- a/.test-infra/jenkins/job_PostCommit_TransformService_Direct.groovy +++ b/.test-infra/jenkins/job_PostCommit_TransformService_Direct.groovy @@ -43,7 +43,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_TransformService_Direct', rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava') commonJobProperties.setGradleSwitches(delegate) - switches '-PcompileAndRunTestsWithJava11' + switches '-PtestJavaVersion=11' switches "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}" switches("-PuseWheelDistribution") switches("-PpythonVersion=${pythonVersion}") diff --git a/build.gradle.kts b/build.gradle.kts index ea1b4e6784e3..5bcfbb3ed06e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -712,14 +712,12 @@ if (project.hasProperty("javaLinkageArtifactIds")) { } } } -if (project.hasProperty("compileAndRunTestsWithJava11")) { - tasks.getByName("javaPreCommitPortabilityApi").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion") - tasks.getByName("javaExamplesDataflowPrecommit").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion") - tasks.getByName("sqlPreCommit").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion") -} else if (project.hasProperty("compileAndRunTestsWithJava17")) { - tasks.getByName("javaPreCommitPortabilityApi").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion17") - tasks.getByName("javaExamplesDataflowPrecommit").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion17") - tasks.getByName("sqlPreCommit").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion17") +if (project.hasProperty("testJavaVersion")) { + var testVer = project.property("testJavaVersion") + + tasks.getByName("javaPreCommitPortabilityApi").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer") + tasks.getByName("javaExamplesDataflowPrecommit").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer") + tasks.getByName("sqlPreCommit").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer") } else { allprojects { tasks.withType(Test::class).configureEach { diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index b13fd00dc249..b32a41f80279 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -946,6 +946,29 @@ class BeamModulePlugin implements Plugin { ] } + project.ext.setJava21Options = { CompileOptions options -> + def java21Home = project.findProperty("java21Home") + options.fork = true + options.forkOptions.javaHome = java21Home as File + options.compilerArgs += ['-Xlint:-path'] + // Error prone requires some packages to be exported/opened for Java 17+ + // Disabling checks since this property is only used for Jenkins tests + // https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support + options.errorprone.errorproneArgs.add("-XepDisableAllChecks") + options.forkOptions.jvmArgs += [ + "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", + "-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", + "-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", + "-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" + ] + } + project.ext.repositories = { maven { name "testPublicationLocal" @@ -1492,7 +1515,7 @@ class BeamModulePlugin implements Plugin { options.errorprone.errorproneArgs.add("-Xep:Slf4jLoggerShouldBeNonStatic:OFF") } - if (project.hasProperty("compileAndRunTestsWithJava11")) { + if (project.findProperty('testJavaVersion') == "11") { def java11Home = project.findProperty("java11Home") project.tasks.compileTestJava { options.fork = true @@ -1504,7 +1527,7 @@ class BeamModulePlugin implements Plugin { useJUnit() executable = "${java11Home}/bin/java" } - } else if (project.hasProperty("compileAndRunTestsWithJava17")) { + } else if (project.findProperty('testJavaVersion') == "17") { def java17Home = project.findProperty("java17Home") project.tasks.compileTestJava { setCompileAndRuntimeJavaVersion(options.compilerArgs, '17') @@ -1514,6 +1537,16 @@ class BeamModulePlugin implements Plugin { useJUnit() executable = "${java17Home}/bin/java" } + } else if (project.findProperty('testJavaVersion') == "21") { + def java21Home = project.findProperty("java21Home") + project.tasks.compileTestJava { + setCompileAndRuntimeJavaVersion(options.compilerArgs, '21') + project.ext.setJava17Options(options) + } + project.tasks.withType(Test).configureEach { + useJUnit() + executable = "${java21Home}/bin/java" + } } if (configuration.shadowClosure) { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java index 31a555989afd..f531b5be344d 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java @@ -94,7 +94,8 @@ public class Environments { public enum JavaVersion { java8("java", "1.8", 8), java11("java11", "11", 11), - java17("java17", "17", 17); + java17("java17", "17", 17), + java21("java21", "21", 21); // Legacy name, as used in container image private final String legacyName; @@ -119,6 +120,7 @@ public String specification() { return this.specification; } + /** Return the LTS java version given the Java specification version. */ public static JavaVersion forSpecification(String specification) { for (JavaVersion ver : JavaVersion.values()) { if (ver.specification.equals(specification)) { @@ -137,7 +139,7 @@ public static JavaVersion forSpecification(String specification) { } } LOG.warn( - "unsupported Java version: {}, falling back to: {}", + "Unsupported Java version: {}, falling back to: {}", specification, fallback.specification); return fallback; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java index ae429fb1fe6d..b71a654f1031 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java @@ -291,6 +291,8 @@ public void testLtsJavaVersion() { assertEquals("java11", JavaVersion.java11.legacyName()); assertEquals(JavaVersion.java17, JavaVersion.forSpecification("17")); assertEquals("java17", JavaVersion.java17.legacyName()); + assertEquals(JavaVersion.java21, JavaVersion.forSpecification("21")); + assertEquals("java21", JavaVersion.java21.legacyName()); } @Test @@ -303,7 +305,9 @@ public void testNonLtsJavaVersion() { assertEquals(JavaVersion.java17, JavaVersion.forSpecification("15")); assertEquals(JavaVersion.java17, JavaVersion.forSpecification("16")); assertEquals(JavaVersion.java17, JavaVersion.forSpecification("18")); - assertEquals(JavaVersion.java17, JavaVersion.forSpecification("19")); + assertEquals(JavaVersion.java21, JavaVersion.forSpecification("19")); + assertEquals(JavaVersion.java21, JavaVersion.forSpecification("20")); + assertEquals(JavaVersion.java21, JavaVersion.forSpecification("21")); } @Test(expected = UnsupportedOperationException.class) diff --git a/runners/google-cloud-dataflow-java/arm/build.gradle b/runners/google-cloud-dataflow-java/arm/build.gradle index e79eeedcd828..71cbc7c58e86 100644 --- a/runners/google-cloud-dataflow-java/arm/build.gradle +++ b/runners/google-cloud-dataflow-java/arm/build.gradle @@ -76,10 +76,8 @@ dependencies { } def javaVer = "java8" -if(project.hasProperty('compileAndRunTestsWithJava17')) { - javaVer = "java17" -} else if(project.hasProperty('compileAndRunTestsWithJava11')) { - javaVer = "java11" +if (project.hasProperty('testJavaVersion')) { + javaVer = "java${project.getProperty('testJavaVersion')}" } def dataflowProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing' def dataflowRegion = project.findProperty('dataflowRegion') ?: 'us-central1' diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index adc1f2e09bc4..e4f34687d31c 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -279,10 +279,8 @@ def createRunnerV2ValidatesRunnerTest = { Map args -> // task ordering such that the registry doesn't get cleaned up prior to task completion. def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") { def javaVer = "java8" - if(project.hasProperty('compileAndRunTestsWithJava17')) { - javaVer = "java17" - } else if(project.hasProperty('compileAndRunTestsWithJava11')) { - javaVer = "java11" + if(project.hasProperty('testJavaVersion')) { + javaVer = "java${project.getProperty('testJavaVersion')}" } dependsOn ":sdks:java:container:${javaVer}:docker" def defaultDockerImageName = containerImageName( diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java index 26548038a1df..891b4c0454c9 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java @@ -398,10 +398,9 @@ && isServiceEndpoint(dataflowOptions.getDataflowEndpoint())) { // Adding the Java version to the SDK name for user's and support convenience. String agentJavaVer = "(JRE 8 environment)"; - if (Environments.getJavaVersion() == Environments.JavaVersion.java17) { - agentJavaVer = "(JRE 17 environment)"; - } else if (Environments.getJavaVersion() == Environments.JavaVersion.java11) { - agentJavaVer = "(JRE 11 environment)"; + if (Environments.getJavaVersion() != Environments.JavaVersion.java8) { + agentJavaVer = + String.format("(JRE %s environment)", Environments.getJavaVersion().specification()); } DataflowRunnerInfo dataflowRunnerInfo = DataflowRunnerInfo.getDataflowRunnerInfo(); diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle index d0dbe453ddfb..74013de6107d 100644 --- a/runners/spark/spark_runner.gradle +++ b/runners/spark/spark_runner.gradle @@ -63,8 +63,9 @@ def sparkTestProperties(overrides = [:]) { def sparkTestJvmArgs() { - // run tests with Java 17 using -PcompileAndRunTestsWithJava17 -Pjava17Home=??? - if (project.hasProperty("compileAndRunTestsWithJava17")) { + // run tests with Java 17 using -PtestJavaVersion=17 -Pjava17Home=??? + if (project.hasProperty('testJavaVersion') && + project.getProperty('testJavaVersion') in ['17', '21']) { return [ "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", // add-opens below required for Kryo FieldSerializer / SparkRunnerKryoRegistratorTest diff --git a/sdks/java/container/Dockerfile b/sdks/java/container/Dockerfile index e0fa8d4a0a6f..9c266ea132b8 100644 --- a/sdks/java/container/Dockerfile +++ b/sdks/java/container/Dockerfile @@ -15,8 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### +ARG base_image ARG java_version -FROM eclipse-temurin:${java_version} +FROM ${base_image}:${java_version} LABEL Author "Apache Beam " ARG TARGETOS ARG TARGETARCH diff --git a/sdks/java/container/agent/build.gradle b/sdks/java/container/agent/build.gradle index df3780e45446..044d175257cc 100644 --- a/sdks/java/container/agent/build.gradle +++ b/sdks/java/container/agent/build.gradle @@ -20,10 +20,13 @@ plugins { id 'org.apache.beam.module' } +// the order is intended here if (project.hasProperty('java11Home')) { javaVersion = "1.11" } else if (project.hasProperty('java17Home')) { javaVersion = "1.17" +} else if (project.hasProperty('java21Home')) { + javaVersion = "1.21" } applyJavaNature( @@ -42,6 +45,7 @@ jar { } } +// the order is intended here if (project.hasProperty('java11Home')) { def java11Home = project.findProperty('java11Home') project.tasks.withType(JavaCompile) { @@ -53,6 +57,14 @@ if (project.hasProperty('java11Home')) { project.tasks.withType(JavaCompile) { setJava17Options(options) + checkerFramework { + skipCheckerFramework = true + } + } +} else if (project.hasProperty('java21Home')) { + project.tasks.withType(JavaCompile) { + setJava21Options(options) + checkerFramework { skipCheckerFramework = true } @@ -62,7 +74,7 @@ if (project.hasProperty('java11Home')) { // Module classes requires JDK > 8 project.tasks.each { it.onlyIf { - project.hasProperty('java11Home') || project.hasProperty('java17Home') + project.hasProperty('java11Home') || project.hasProperty('java17Home') || project.hasProperty('java21Home') || JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 } } diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle index cc427494ed6e..6be531bc5e80 100644 --- a/sdks/java/container/common.gradle +++ b/sdks/java/container/common.gradle @@ -29,6 +29,7 @@ applyDockerNature() if (!project.hasProperty('imageJavaVersion')) { throw new GradleException('imageJavaVersion project property must be set') } +def javaBaseImage = project.findProperty('javaBaseImage') ?: 'eclipse-temurin' def imageJavaVersion = project.findProperty('imageJavaVersion') description = "Apache Beam :: SDKs :: Java :: Container :: Java ${imageJavaVersion} Container" @@ -71,19 +72,19 @@ task copySdkHarnessLauncher(type: Copy) { } task copyJavaThirdPartyLicenses(type: Copy) { - from("${project(':sdks:java:container').buildDir}/target/third_party_licenses") + from project(':sdks:java:container').layout.buildDirectory.dir('target/third_party_licenses') into "build/target/third_party_licenses" dependsOn ':sdks:java:container:pullLicenses' } task copyGolangLicenses(type: Copy) { - from "${project(':release:go-licenses:java').buildDir}/output" + from project(':release:go-licenses:java').layout.buildDirectory.dir('output') into "build/target/go-licenses" dependsOn ':release:go-licenses:java:createLicenses' } task copyJdkOptions(type: Copy) { - if (imageJavaVersion == "17" || imageJavaVersion == "11") { + if (["11", "17", "21"].contains(imageJavaVersion)) { from "option-jamm.json" } from "java${imageJavaVersion}-security.properties" @@ -96,11 +97,25 @@ task skipPullLicenses(type: Exec) { args "-c", "mkdir -p build/target/go-licenses build/target/options build/target/third_party_licenses && touch build/target/go-licenses/skip && touch build/target/third_party_licenses/skip" } +// Java11+ container depends on the java agent project. To compile it, need a compatible JDK version: +// lower bound 11 and upper bound imageJavaVersion task validateJavaHome { - if (imageJavaVersion == "11" || imageJavaVersion == "17") { + def requiredForVer = ["11", "17", "21"] + if (requiredForVer.contains(imageJavaVersion)) { doFirst { - if (!project.hasProperty('java17Home') && !project.hasProperty('java11Home')) { - throw new GradleException('java17Home or java11Home property required. Re-run with -Pjava17Home or -Pjava11Home') + boolean propertyFound = false + // enable to build agent with compatible java versions (11-requiredForVer) + for (def checkVer : requiredForVer) { + if (project.hasProperty("java${checkVer}Home")) { + propertyFound = true + } + if (checkVer == imageJavaVersion) { + // cannot build agent with a higher version than the docker java ver + break + } + } + if (!propertyFound) { + throw new GradleException("java${imageJavaVersion}Home or compatible properties required for imageJavaVersion=${imageJavaVersion}. Re-run with -Pjava${imageJavaVersion}Home") } } } @@ -124,6 +139,7 @@ docker { buildArgs([ 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"]), + 'base_image': javaBaseImage, 'java_version': imageJavaVersion, ]) buildx useBuildx diff --git a/sdks/java/container/java21/build.gradle b/sdks/java/container/java21/build.gradle new file mode 100644 index 000000000000..038064102dcb --- /dev/null +++ b/sdks/java/container/java21/build.gradle @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +project.ext { + // TODO(https://github.com/apache/beam/issues/28120) switch to temurin once available + javaBaseImage = 'openjdk' + imageJavaVersion = '21' +} + +// Load the main build script which contains all build logic. +apply from: "../common.gradle" + +dependencies { + dockerDependency project(path: ":sdks:java:container:agent") +} \ No newline at end of file diff --git a/sdks/java/container/java21/option-jamm.json b/sdks/java/container/java21/option-jamm.json new file mode 100644 index 000000000000..5647ff66be5c --- /dev/null +++ b/sdks/java/container/java21/option-jamm.json @@ -0,0 +1,12 @@ +{ + "name": "jamm", + "enabled": true, + "options": { + "java_arguments": [ + "--add-modules=jamm", + "--module-path=/opt/apache/beam/jars/jamm.jar", + "--add-opens=java.base/java.lang=jamm", + "--add-opens=java.base/java.util=jamm" + ] + } +} \ No newline at end of file diff --git a/sdks/java/testing/jpms-tests/build.gradle b/sdks/java/testing/jpms-tests/build.gradle index 6321f874c903..429fc063027f 100644 --- a/sdks/java/testing/jpms-tests/build.gradle +++ b/sdks/java/testing/jpms-tests/build.gradle @@ -23,10 +23,10 @@ plugins { } // overwrite javaVersion before applyJavaNature -if (project.hasProperty("compileAndRunTestsWithJava17")) { - javaVersion = '1.17' +if (project.hasProperty("testJavaVersion")) { + javaVersion = "1.${project.getProperty('testJavaVersion')}" as String } else { - javaVersion = '1.11' + javaVersion = "1.11" } applyJavaNature( @@ -42,13 +42,14 @@ ext.summary = "E2E test for Java 9 modules" // direct compileJava to use specified java version. project.tasks.compileJava { - if (project.hasProperty("compileAndRunTestsWithJava11")) { + if (project.hasProperty('testJavaVersion')) { options.fork = true - options.forkOptions.javaHome = project.findProperty("java11Home") as File - } else if (project.hasProperty("compileAndRunTestsWithJava17")) { - options.fork = true - options.forkOptions.javaHome = project.findProperty("java17Home") as File - setJava17Options(options) + options.forkOptions.javaHome = project.findProperty("java${project.getProperty('testJavaVersion')}Home") as File + if (project.getProperty('testJavaVersion') == '17') { + setJava17Options(options) + } else if (project.getProperty('testJavaVersion') == '21') { + setJava21Options(options) + } } } @@ -117,10 +118,12 @@ plugins.withType(JavaPlugin).configureEach{ } } -// JPMS requires JDK > 8 +// JPMS requires JDK > 8. Test tasks enabled when either +// (i) testJavaVersion property specified (assumed to be >8) or; +// (ii) current Java version is greater than 8 project.tasks.each { it.onlyIf { - project.hasProperty("compileAndRunTestsWithJava17") + project.hasProperty('testJavaVersion') || JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 } } diff --git a/sdks/java/testing/test-utils/build.gradle b/sdks/java/testing/test-utils/build.gradle index 50c815dd57f7..6e30693d8894 100644 --- a/sdks/java/testing/test-utils/build.gradle +++ b/sdks/java/testing/test-utils/build.gradle @@ -43,24 +43,15 @@ dependencies { testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadowTest") } -task verifyJavaVersion(type: Test) { - filter { - includeTestsMatching 'org.apache.beam.sdk.testutils.jvmverification.JvmVerification.verifyCodeIsCompiledWithJava8' - includeTestsMatching 'org.apache.beam.sdk.testutils.jvmverification.JvmVerification.verifyTestCodeIsCompiledWithJava11' - includeTestsMatching 'org.apache.beam.sdk.testutils.jvmverification.JvmVerification.verifyRunningJVMVersionIs11' - } - doLast { - println 'Java verified' +['11', '17', '21'].each { + tasks.create(name: "verifyJavaVersion${it}", type: Test) { + filter { + includeTestsMatching "org.apache.beam.sdk.testutils.jvmverification.JvmVerification.verifyCodeIsCompiledWithJava8" + includeTestsMatching "org.apache.beam.sdk.testutils.jvmverification.JvmVerification.verifyTestCodeIsCompiledWithJava${it}" + includeTestsMatching "org.apache.beam.sdk.testutils.jvmverification.JvmVerification.verifyRunningJVMVersionIs${it}" + } + doLast { + println 'Java verified' + } } } - -task verifyJavaVersion17(type: Test) { - filter { - includeTestsMatching 'org.apache.beam.sdk.testutils.jvmverification.JvmVerification.verifyCodeIsCompiledWithJava8' - includeTestsMatching 'org.apache.beam.sdk.testutils.jvmverification.JvmVerification.verifyTestCodeIsCompiledWithJava17' - includeTestsMatching 'org.apache.beam.sdk.testutils.jvmverification.JvmVerification.verifyRunningJVMVersionIs17' - } - doLast { - println 'Java verified' - } -} \ No newline at end of file diff --git a/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java b/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java index ad29e8b6a1d6..a6b5d6dca6c1 100644 --- a/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java +++ b/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java @@ -20,6 +20,7 @@ import static org.apache.beam.sdk.testutils.jvmverification.JvmVerification.Java.v11; import static org.apache.beam.sdk.testutils.jvmverification.JvmVerification.Java.v17; import static org.apache.beam.sdk.testutils.jvmverification.JvmVerification.Java.v1_8; +import static org.apache.beam.sdk.testutils.jvmverification.JvmVerification.Java.v21; import static org.junit.Assert.assertEquals; import java.io.IOException; @@ -39,6 +40,7 @@ public class JvmVerification { versionMapping.put("0034", v1_8); versionMapping.put("0037", v11); versionMapping.put("003d", v17); + versionMapping.put("0041", v21); } // bytecode @@ -62,6 +64,11 @@ public void verifyTestCodeIsCompiledWithJava17() throws IOException { assertEquals(v17, getByteCodeVersion(JvmVerification.class)); } + @Test + public void verifyTestCodeIsCompiledWithJava21() throws IOException { + assertEquals(v21, getByteCodeVersion(JvmVerification.class)); + } + // jvm @Test public void verifyRunningJVMVersionIs11() { @@ -75,6 +82,12 @@ public void verifyRunningJVMVersionIs17() { assertEquals(v17.name, version); } + @Test + public void verifyRunningJVMVersionIs21() { + final String version = getJavaSpecification(); + assertEquals(v21.name, version); + } + private static Java getByteCodeVersion(final Class clazz) throws IOException { final InputStream stream = clazz.getClassLoader().getResourceAsStream(clazz.getName().replace(".", "/") + ".class"); @@ -91,7 +104,8 @@ private static String getJavaSpecification() { enum Java { v1_8("1.8"), v11("11"), - v17("17"); + v17("17"), + v21("21"); final String name; diff --git a/settings.gradle.kts b/settings.gradle.kts index d833a323212c..e8e374eb6e19 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -175,6 +175,7 @@ include(":sdks:java:container:agent") include(":sdks:java:container:java8") include(":sdks:java:container:java11") include(":sdks:java:container:java17") +include(":sdks:java:container:java21") include(":sdks:java:core") include(":sdks:java:core:jmh") include(":sdks:java:expansion-service") diff --git a/website/www/site/content/en/roadmap/java-sdk.md b/website/www/site/content/en/roadmap/java-sdk.md index b65424b57a3d..a1c85e139193 100644 --- a/website/www/site/content/en/roadmap/java-sdk.md +++ b/website/www/site/content/en/roadmap/java-sdk.md @@ -17,9 +17,9 @@ limitations under the License. # Java SDK Roadmap -## Next Java LTS version support (Java 17) +## Next Java LTS version support (Java 21) Work to support the next LTS release of Java is in progress. For more details -about the scope and info on the various tasks please see the JIRA ticket. +about the scope and info on the various tasks please see the GitHub Issue. -- JIRA: [BEAM-12240](https://issues.apache.org/jira/browse/BEAM-12240) +- GitHub: [#28120](https://github.com/apache/beam/issues/28120)