Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix XVR_Direct time out #25247

Merged
merged 1 commit into from
Feb 1, 2023
Merged

Fix XVR_Direct time out #25247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Direct',
description('Runs the CrossLanguageValidatesRunner suite on the Direct runner.')

// Set common parameters.
commonJobProperties.setTopLevelMainJobProperties(delegate)
commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)

// Publish all test results to Jenkins
publishers {
Expand All @@ -43,6 +43,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Direct',
tasks(':sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner')
commonJobProperties.setGradleSwitches(delegate)
switches("-PpythonVersion=${pythonVersion}")
// only run non-python task (e.g. GoUsingJava) once
switches("-PskipNonPythonTask=${pythonVersion != CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS[0]}")
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions .test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Flink.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Flink',
tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:job-server:validatesCrossLanguageRunner")
commonJobProperties.setGradleSwitches(delegate)
switches("-PpythonVersion=${pythonVersion}")
// only run non-python task (e.g. GoUsingJava) once
switches("-PskipNonPythonTask=${pythonVersion != CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS[0]}")
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions .test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Samza.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Samza',
tasks(":runners:samza:job-server:validatesCrossLanguageRunner")
commonJobProperties.setGradleSwitches(delegate)
switches("-PpythonVersion=${pythonVersion}")
// only run non-python task (e.g. GoUsingJava) once
switches("-PskipNonPythonTask=${pythonVersion != CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS[0]}")
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions .test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Spark.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Spark',
tasks(':runners:spark:2:job-server:validatesCrossLanguageRunner')
commonJobProperties.setGradleSwitches(delegate)
switches("-PpythonVersion=${pythonVersion}")
// only run non-python task (e.g. GoUsingJava) once
switches("-PskipNonPythonTask=${pythonVersion != CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS[0]}")
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions .test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Spark3.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Spark3',
tasks(':runners:spark:3:job-server:validatesCrossLanguageRunner')
commonJobProperties.setGradleSwitches(delegate)
switches("-PpythonVersion=${pythonVersion}")
// only run non-python task (e.g. GoUsingJava) once
switches("-PskipNonPythonTask=${pythonVersion != CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS[0]}")
}
}
}
Expand Down
14 changes: 9 additions & 5 deletions buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2392,7 +2392,7 @@ class BeamModulePlugin implements Plugin<Project> {
"python_port": pythonPort
]
def serviceArgs = project.project(':sdks:python').mapToArgString(expansionServiceOpts)
def pythonContainerSuffix = project.project(':sdks:python').pythonVersion == '2.7' ? '2' : project.project(':sdks:python').pythonVersion.replace('.', '')
def pythonContainerSuffix = project.project(':sdks:python').pythonVersion.replace('.', '')
def javaContainerSuffix
if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
javaContainerSuffix = 'java8'
Expand Down Expand Up @@ -2534,9 +2534,13 @@ class BeamModulePlugin implements Plugin<Project> {
dependsOn setupTask
dependsOn config.startJobServer
}
mainTask.configure{dependsOn goTask}
cleanupTask.configure{mustRunAfter goTask}
config.cleanupJobServer.configure{mustRunAfter goTask}
// CrossLanguageValidatesRunnerTask is setup under python sdk but also runs tasks not involving
// python versions. set 'skipNonPythonTask' property to avoid duplicated run of these tasks.
if (!(project.hasProperty('skipNonPythonTask') && project.skipNonPythonTask == 'true')) {
mainTask.configure { dependsOn goTask }
}
cleanupTask.configure { mustRunAfter goTask }
config.cleanupJobServer.configure { mustRunAfter goTask }
Comment on lines +2542 to +2543
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to special case these as well so that they run even if we're not depending on the goTask?

Copy link
Contributor Author

@Abacn Abacn Feb 1, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, besides being a dependency to validatesCrossLanguageRunner, validatesCrossLanguageRunnerGoUsingJava task can be triggered alone, where we are doing this for Dataflow XVR postcommits. Then clean up tassks should also be configured to run after it.

Copy link
Contributor Author

@Abacn Abacn Feb 1, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, answered wrong the yes/no question. I mean run after requirements are always configured not relevant to the dependency of validatesCrossLanguageRunnerGoUsingJava -> validatesCrossLanguageRunner, so I consider the answer is no

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see - thanks!

}

/** ***********************************************************************************************/
Expand Down Expand Up @@ -2795,7 +2799,7 @@ class BeamModulePlugin implements Plugin<Project> {
}

project.ext.getVersionSuffix = { String version ->
return version == '2.7' ? '2' : version.replace('.', '')
return version.replace('.', '')
}

project.ext.getVersionsAsList = { String propertyName ->
Expand Down
2 changes: 1 addition & 1 deletion sdks/python/test-suites/direct/common.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
*/

def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '')
def pythonContainerVersion = project.ext.pythonVersion == '2.7' ? '2' : project.ext.pythonVersion
def pythonContainerVersion = project.ext.pythonVersion
def runScriptsDir = "${rootDir}/sdks/python/scripts"
// Basic test options for ITs running on Jenkins.
def basicTestOpts = [
Expand Down