Fixed beam_PreCommit_Flink_Container.yml (#33219)

* Fixed beam_PreCommit_Flink_Container.yml

* Update beam_PreCommit_Flink_Container.yml

* Update beam_PreCommit_Flink_Container.yml

* refactored the options

* added test type

* fixed the python gradle

* Added the python version

* Fixed the java test

* fixed java options

* fixed options

* fixed the options

* fixed the job name

.github/workflows/beam_PreCommit_Flink_Container.yml

@@ -98,6 +98,21 @@ jobs:
           comment_phrase: ${{ matrix.job_phrase }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+      - name: Setup environment
+        uses: ./.github/actions/setup-environment-action
+        with:
+          python-version: default
+      - name: Prepare test arguments
+        uses: ./.github/actions/test-arguments-action
+        with:
+          test-type: precommit
+          test-language: go,python,java
+          argument-file-paths: |
+            ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt
+            ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt
+            ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt
+      - name: get current time
+        run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV
       - name: Start Flink with 2 workers
         env:
           FLINK_NUM_WORKERS: 2
@@ -112,36 +127,18 @@ jobs:
           arguments: |
             -PloadTest.mainClass=combine \
             -Prunner=PortableRunner \
-            -PloadTest.args="
-              --runner=FlinkRunner \
-              --job_endpoint=localhost:8099 \
-              --environment_type=DOCKER \
-              --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest \
-              --parallelism=1 \
-              --input_options=\"{\"num_records\":200,\"key_size\":1,\"value_size\":9}\"
-              --fanout=1 \
-              --top_count=10 \
-              --iterations=1"
+            '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_1 }} --job_name=flink-tests-go-${{env.NOW_UTC}}'
 
       # Run a Python Combine load test to verify the Flink container
       - name: Run Flink Container Test with Python Combine
         timeout-minutes: 10
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :sdks:python:test:load:run
+          gradle-command: :sdks:python:apache_beam:testing:load_tests:run
           arguments: |
             -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
             -Prunner=FlinkRunner \
-            -PloadTest.args="
-              --runner=PortableRunner \
-              --job_endpoint=localhost:8099 \
-              --environment_type=DOCKER \
-              --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest \
-              --parallelism=1 \
-              --input_options=\"{\"num_records\":200,\"key_size\":1,\"value_size\":9,\"algorithm\":\"lcg\"}\" \
-              --fanout=1 \
-              --top_count=10 \
-              --iterations=1"
+            '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_2 }} --job_name=flink-tests-python-${{env.NOW_UTC}}'
 
       # Run a Java Combine load test to verify the Flink container
       - name: Run Flink Container Test with Java Combine
@@ -152,17 +149,9 @@ jobs:
           arguments: |
             -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \
             -Prunner=:runners:flink:1.17 \
-            -PloadTest.args="
-              --runner=FlinkRunner \
-              --environment_type=DOCKER \
-              --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_java11_sdk:latest \
-              --parallelism=1 \
-              --sourceOptions={\"numRecords\":200,\"keySizeBytes\":1,\"valueSizeBytes\":9} \
-              --fanout=1 \
-              --iterations=1 \
-              --topCount=10"
+            '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_3 }} --jobName=flink-tests-java11-${{env.NOW_UTC}}'
 
       - name: Teardown Flink
         if: always()
         run: |
-          ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete
+          ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete

.github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt

@@ -0,0 +1,24 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+--input_options=''{\"num_records\":200,\"key_size\":1,\"value_size\":9}''
+--fanout=1
+--top_count=10
+--parallelism=2
+--endpoint=localhost:8099
+--environment_type=DOCKER
+--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
+--runner=FlinkRunner

.github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt

@@ -0,0 +1,25 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+
+--sourceOptions={"numRecords":200,"keySizeBytes":1,"valueSizeBytes":9}
+--fanout=1
+--iterations=1
+--topCount=10
+--parallelism=2
+--jobEndpoint=localhost:8099
+--defaultEnvironmentType=DOCKER
+--defaultEnvironmentConfig=gcr.io/apache-beam-testing/beam-sdk/beam_java11_sdk:latest
+--runner=FlinkRunner

.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt

@@ -0,0 +1,23 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+--input_options=''{\\"num_records\\":200,\\"key_size\\":1,\\"value_size\\":9,\\"algorithm\\":\\"lcg\\"}''
+--parallelism=2
+--job_endpoint=localhost:8099
+--environment_type=DOCKER
+--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest
+--top_count=10
+--runner=PortableRunner