Skip to content

Commit

Permalink
Support PodAnnotations & Labels to cronjob pod template spec
Browse files Browse the repository at this point in the history 
 - Add support to add labels as well as pod annotations to the underlying pod template spec metadata
 - Add tests

Resolves #445
Resolves #425

Fix test

Fix task launcher

Polishing on merge
  • Loading branch information
@ilayaperumalg
ilayaperumalg authored and Glenn Renfro committed May 26, 2021
1 parent 1505ca7 commit 52bd1ee
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 9 deletions.
10 changes: 7 additions & 3 deletions .../java/org/springframework/cloud/deployer/spi/kubernetes/DeploymentPropertiesResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -613,8 +613,13 @@ Map<String, String> getDeploymentLabels(Map<String, String> kubernetesDeployerPr
String deploymentLabels = PropertyParserUtils.getDeploymentPropertyValue(kubernetesDeployerProperties,
this.propertyPrefix + ".deploymentLabels", "");

if (StringUtils.hasText(deploymentLabels)) {
String[] deploymentLabel = deploymentLabels.split(",");
// Add deployment labels set at the deployer level.
String updatedLabels = StringUtils.hasText(this.properties.getDeploymentLabels()) ?
new StringBuilder().append(deploymentLabels).append(StringUtils.hasText(deploymentLabels) ? ",": "")
.append(this.properties.getDeploymentLabels()).toString() : deploymentLabels;

if (StringUtils.hasText(updatedLabels)) {
String[] deploymentLabel = updatedLabels.split(",");

for (String label : deploymentLabel) {
String[] labelPair = label.split(":");
Expand All @@ -623,7 +628,6 @@ Map<String, String> getDeploymentLabels(Map<String, String> kubernetesDeployerPr
labels.put(labelPair[0].trim(), labelPair[1].trim());
}
}

return labels;
}

Expand Down
13 changes: 13 additions & 0 deletions .../java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesDeployerProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -795,6 +795,11 @@ public void setEnvironmentVariables(String[] environmentVariables) {
*/
private List<Container> additionalContainers;

/**
* Deployment label to be applied to Deployment, StatefulSet, JobSpec etc.,
*/
private String deploymentLabels;

public String getNamespace() {
return namespace;
}
Expand Down Expand Up @@ -1451,4 +1456,12 @@ Lifecycle getLifecycle() {
void setLifecycle(Lifecycle lifecycle) {
this.lifecycle = lifecycle;
}

public String getDeploymentLabels() {
return deploymentLabels;
}

public void setDeploymentLabels(String deploymentLabels) {
this.deploymentLabels = deploymentLabels;
}
}
12 changes: 7 additions & 5 deletions src/main/java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesScheduler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package org.springframework.cloud.deployer.spi.kubernetes;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -50,7 +49,7 @@
* @author Ilayaperumal Gopinathan
*/
public class KubernetesScheduler extends AbstractKubernetesDeployer implements Scheduler {
private static final String SPRING_CRONJOB_ID_KEY = "spring-cronjob-id";
protected static final String SPRING_CRONJOB_ID_KEY = "spring-cronjob-id";

private static final String SCHEDULE_EXPRESSION_FIELD_NAME = "spec.schedule";

Expand Down Expand Up @@ -167,8 +166,8 @@ public List<ScheduleInfo> list() {
}

protected CronJob createCronJob(ScheduleRequest scheduleRequest) {
Map<String, String> labels = Collections.singletonMap(SPRING_CRONJOB_ID_KEY,
scheduleRequest.getDefinition().getName());
Map<String, String> labels = new HashMap<>();
labels.put(SPRING_CRONJOB_ID_KEY, scheduleRequest.getDefinition().getName());

Map<String, String> schedulerProperties = scheduleRequest.getSchedulerProperties();
String schedule = schedulerProperties.get(SchedulerPropertyKeys.CRON_EXPRESSION);
Expand All @@ -179,11 +178,14 @@ protected CronJob createCronJob(ScheduleRequest scheduleRequest) {
if (StringUtils.hasText(taskServiceAccountName)) {
podSpec.setServiceAccountName(taskServiceAccountName);
}
Map<String, String> annotations = this.deploymentPropertiesResolver.getPodAnnotations(scheduleRequest.getSchedulerProperties());
labels.putAll(this.deploymentPropertiesResolver.getDeploymentLabels(scheduleRequest.getSchedulerProperties()));

CronJob cronJob = new CronJobBuilder().withNewMetadata().withName(scheduleRequest.getScheduleName())
.withLabels(labels).withAnnotations(this.deploymentPropertiesResolver.getJobAnnotations(schedulerProperties)).endMetadata()
.withNewSpec().withSchedule(schedule).withNewJobTemplate()
.withNewSpec().withNewTemplate().withSpec(podSpec).endTemplate().endSpec()
.withNewSpec().withNewTemplate().withNewMetadata().addToAnnotations(annotations).addToLabels(labels)
.endMetadata().withSpec(podSpec).endTemplate().endSpec()
.endJobTemplate().endSpec().build();

setImagePullSecret(scheduleRequest, cronJob);
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesTaskLauncher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ private void launch(String appId, AppDeploymentRequest request) {
.addToLabels(idMap)
.addToLabels(deploymentLabels)
.withAnnotations(this.deploymentPropertiesResolver.getJobAnnotations(deploymentProperties))
.addToAnnotations(this.deploymentPropertiesResolver.getPodAnnotations(deploymentProperties))
.build();
PodTemplateSpec podTemplateSpec = new PodTemplateSpec(objectMeta, podSpec);

Expand All @@ -265,13 +266,15 @@ private void launch(String appId, AppDeploymentRequest request) {
}
else {
logger.debug(String.format("Launching Pod for task: %s", appId));

this.client.pods()
.createNew()
.withNewMetadata()
.withName(appId)
.withLabels(podLabelMap)
.addToLabels(deploymentLabels)
.withAnnotations(this.deploymentPropertiesResolver.getJobAnnotations(deploymentProperties))
.addToAnnotations(this.deploymentPropertiesResolver.getPodAnnotations(deploymentProperties))
.addToLabels(idMap)
.endMetadata()
.withSpec(podSpec)
Expand Down
69 changes: 68 additions & 1 deletion src/test/java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesSchedulerIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,9 +505,11 @@ public void testImagePullPolicyOverride() {
}

@Test
public void testJobAnnotationsFromSchedulerProperties() {
public void testJobAnnotationsAndLabelsFromSchedulerProperties() {
KubernetesSchedulerProperties kubernetesSchedulerProperties = new KubernetesSchedulerProperties();
kubernetesSchedulerProperties.setJobAnnotations("test1:value1");
kubernetesSchedulerProperties.setPodAnnotations("podtest1:podvalue1");
kubernetesSchedulerProperties.setDeploymentLabels("label1:value1,label2:value2");
if (kubernetesSchedulerProperties.getNamespace() == null) {
kubernetesSchedulerProperties.setNamespace("default");
}
Expand All @@ -524,6 +526,71 @@ public void testJobAnnotationsFromSchedulerProperties() {
CronJob cronJob = kubernetesScheduler.createCronJob(scheduleRequest);

assertThat(cronJob.getMetadata().getAnnotations().get("test1")).as("Job annotation is not set").isEqualTo("value1");
assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getAnnotations().get("podtest1")).as("Pod annotation is not set").isEqualTo("podvalue1");
assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getLabels().get("label1")).as("Pod Label1 is not set").isEqualTo("value1");
assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getLabels().get("label2")).as("Pod Label2 is not set").isEqualTo("value2");

kubernetesScheduler.unschedule(cronJob.getMetadata().getName());
}

@Test
public void testDefaultLabel() {
KubernetesSchedulerProperties kubernetesSchedulerProperties = new KubernetesSchedulerProperties();
if (kubernetesSchedulerProperties.getNamespace() == null) {
kubernetesSchedulerProperties.setNamespace("default");
}
KubernetesClient kubernetesClient = new DefaultKubernetesClient()
.inNamespace(kubernetesSchedulerProperties.getNamespace());

KubernetesScheduler kubernetesScheduler = new KubernetesScheduler(kubernetesClient,
kubernetesSchedulerProperties);

AppDefinition appDefinition = new AppDefinition(randomName(), getAppProperties());
ScheduleRequest scheduleRequest = new ScheduleRequest(appDefinition, getSchedulerProperties(),
null, getCommandLineArgs(), randomName(), testApplication());

CronJob cronJob = kubernetesScheduler.createCronJob(scheduleRequest);

assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getAnnotations()).isNull();
assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getLabels().size()).as("Should have one label").isEqualTo(1);
assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getLabels().get(
KubernetesScheduler.SPRING_CRONJOB_ID_KEY)).as("Default label is not set").isNotNull();
kubernetesScheduler.unschedule(cronJob.getMetadata().getName());
}

@Test
public void testJobAnnotationsAndLabelsFromSchedulerRequest() {
KubernetesSchedulerProperties kubernetesSchedulerProperties = new KubernetesSchedulerProperties();
kubernetesSchedulerProperties.setJobAnnotations("test1:value1");
kubernetesSchedulerProperties.setPodAnnotations("podtest1:podvalue1");
kubernetesSchedulerProperties.setDeploymentLabels("label1:value1,label2:value2");
if (kubernetesSchedulerProperties.getNamespace() == null) {
kubernetesSchedulerProperties.setNamespace("default");
}
KubernetesClient kubernetesClient = new DefaultKubernetesClient()
.inNamespace(kubernetesSchedulerProperties.getNamespace());

KubernetesScheduler kubernetesScheduler = new KubernetesScheduler(kubernetesClient,
kubernetesSchedulerProperties);

AppDefinition appDefinition = new AppDefinition(randomName(), getAppProperties());
Map<String, String> scheduleProperties = new HashMap<>();
scheduleProperties.putAll(getSchedulerProperties());
scheduleProperties.put("spring.cloud.scheduler.kubernetes.deploymentLabels", "requestLabel1:requestValue1,requestLabel2:requestValue2");
scheduleProperties.put("spring.cloud.scheduler.kubernetes.podAnnotations", "requestPod1:requestPodValue1");
ScheduleRequest scheduleRequest = new ScheduleRequest(appDefinition, scheduleProperties,
null, getCommandLineArgs(), randomName(), testApplication());

CronJob cronJob = kubernetesScheduler.createCronJob(scheduleRequest);

assertThat(cronJob.getMetadata().getAnnotations().get("test1")).as("Job annotation is not set").isEqualTo("value1");
// Pod annotation from the request should override the top level property values
assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getAnnotations().get("requestPod1")).as("Pod annotation is not set").isEqualTo("requestPodValue1");
// Deployment label from the request should get appended to the top level property values
assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getLabels().get("requestLabel1")).as("Pod Label1 from the request is not set").isEqualTo("requestValue1");
assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getLabels().get("requestLabel2")).as("Pod Label2 from the request is not set").isEqualTo("requestValue2");
assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getLabels().get("label1")).as("Pod Label1 is not set").isEqualTo("value1");
assertThat(cronJob.getSpec().getJobTemplate().getSpec().getTemplate().getMetadata().getLabels().get("label2")).as("Pod Label2 is not set").isEqualTo("value2");

kubernetesScheduler.unschedule(cronJob.getMetadata().getName());
}
Expand Down

0 comments on commit 52bd1ee

Please sign in to comment.