From 6a6b010a91abfdad2ac139391bfa8d54f4c50cd3 Mon Sep 17 00:00:00 2001 From: elanv Date: Sun, 21 Feb 2021 02:28:19 +0900 Subject: [PATCH 1/6] improve update and savepoint stability --- api/v1beta1/flinkcluster_default.go | 4 + api/v1beta1/flinkcluster_default_test.go | 56 ++- api/v1beta1/flinkcluster_types.go | 23 +- api/v1beta1/zz_generated.deepcopy.go | 23 +- .../flinkoperator.k8s.io_flinkclusters.yaml | 461 ++++++++++-------- .../batchscheduler/volcano/volcano_test.go | 6 +- controllers/flinkcluster_converter.go | 22 +- controllers/flinkcluster_converter_test.go | 4 +- controllers/flinkcluster_observer.go | 16 +- controllers/flinkcluster_reconciler.go | 89 ++-- controllers/flinkcluster_submit_job_script.go | 2 +- controllers/flinkcluster_updater.go | 184 +++---- controllers/flinkcluster_util.go | 55 ++- controllers/flinkcluster_util_test.go | 79 ++- controllers/model/model.go | 8 +- docs/crd.md | 4 +- .../templates/flink-cluster-crd.yaml | 4 +- 17 files changed, 588 insertions(+), 452 deletions(-) diff --git a/api/v1beta1/flinkcluster_default.go b/api/v1beta1/flinkcluster_default.go index e1d419d9..fbe3d17a 100644 --- a/api/v1beta1/flinkcluster_default.go +++ b/api/v1beta1/flinkcluster_default.go @@ -128,6 +128,10 @@ func _SetJobDefault(jobSpec *JobSpec) { AfterJobCancelled: CleanupActionDeleteCluster, } } + if jobSpec.SavepointMaxAgeForUpdateSeconds == nil { + jobSpec.SavepointMaxAgeForUpdateSeconds = new(int32) + *jobSpec.SavepointMaxAgeForUpdateSeconds = 300 + } } func _SetHadoopConfigDefault(hadoopConfig *HadoopConfig) { diff --git a/api/v1beta1/flinkcluster_default_test.go b/api/v1beta1/flinkcluster_default_test.go index 938affd4..397d46fc 100644 --- a/api/v1beta1/flinkcluster_default_test.go +++ b/api/v1beta1/flinkcluster_default_test.go @@ -45,6 +45,7 @@ func TestSetDefault(t *testing.T) { var defaultJmBlobPort = int32(6124) var defaultJmQueryPort = int32(6125) var defaultJmUIPort = int32(8081) + var defaultJmIngressTLSUse = false var defaultTmDataPort = int32(6121) var defaultTmRPCPort = int32(6122) var defaultTmQueryPort = int32(6125) @@ -52,11 +53,10 @@ func TestSetDefault(t *testing.T) { var defaultJobParallelism = int32(1) var defaultJobNoLoggingToStdout = false var defaultJobRestartPolicy = JobRestartPolicyNever - var defatulJobManagerIngressTLSUse = false + var defaultJobSavepointMaxAgeForUpdateSeconds = int32(300) var defaultMemoryOffHeapRatio = int32(25) var defaultMemoryOffHeapMin = resource.MustParse("600M") - defaultRecreateOnUpdate := new(bool) - *defaultRecreateOnUpdate = true + var defaultRecreateOnUpdate = true var expectedCluster = FlinkCluster{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{}, @@ -70,7 +70,7 @@ func TestSetDefault(t *testing.T) { Replicas: &defaultJmReplicas, AccessScope: "Cluster", Ingress: &JobManagerIngressSpec{ - UseTLS: &defatulJobManagerIngressTLSUse, + UseTLS: &defaultJmIngressTLSUse, }, Ports: JobManagerPorts{ RPC: &defaultJmRPCPort, @@ -99,10 +99,11 @@ func TestSetDefault(t *testing.T) { SecurityContext: nil, }, Job: &JobSpec{ - AllowNonRestoredState: &defaultJobAllowNonRestoredState, - Parallelism: &defaultJobParallelism, - NoLoggingToStdout: &defaultJobNoLoggingToStdout, - RestartPolicy: &defaultJobRestartPolicy, + AllowNonRestoredState: &defaultJobAllowNonRestoredState, + Parallelism: &defaultJobParallelism, + NoLoggingToStdout: &defaultJobNoLoggingToStdout, + RestartPolicy: &defaultJobRestartPolicy, + SavepointMaxAgeForUpdateSeconds: &defaultJobSavepointMaxAgeForUpdateSeconds, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: "DeleteCluster", AfterJobFails: "KeepCluster", @@ -115,7 +116,7 @@ func TestSetDefault(t *testing.T) { MountPath: "/etc/hadoop/conf", }, EnvVars: nil, - RecreateOnUpdate: defaultRecreateOnUpdate, + RecreateOnUpdate: &defaultRecreateOnUpdate, }, Status: FlinkClusterStatus{}, } @@ -134,6 +135,7 @@ func TestSetNonDefault(t *testing.T) { var jmBlobPort = int32(8124) var jmQueryPort = int32(8125) var jmUIPort = int32(9081) + var jmIngressTLSUse = true var tmDataPort = int32(8121) var tmRPCPort = int32(8122) var tmQueryPort = int32(8125) @@ -141,16 +143,15 @@ func TestSetNonDefault(t *testing.T) { var jobParallelism = int32(2) var jobNoLoggingToStdout = true var jobRestartPolicy = JobRestartPolicyFromSavepointOnFailure - var jobManagerIngressTLSUse = true + var jobSavepointMaxAgeForUpdateSeconds = int32(1000) var memoryOffHeapRatio = int32(50) var memoryOffHeapMin = resource.MustParse("600M") + var recreateOnUpdate = false var securityContextUserGroup = int64(9999) var securityContext = corev1.PodSecurityContext{ RunAsUser: &securityContextUserGroup, RunAsGroup: &securityContextUserGroup, } - defaultRecreateOnUpdate := new(bool) - *defaultRecreateOnUpdate = true var cluster = FlinkCluster{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{}, @@ -164,7 +165,7 @@ func TestSetNonDefault(t *testing.T) { Replicas: &jmReplicas, AccessScope: "Cluster", Ingress: &JobManagerIngressSpec{ - UseTLS: &jobManagerIngressTLSUse, + UseTLS: &jmIngressTLSUse, }, Ports: JobManagerPorts{ RPC: &jmRPCPort, @@ -193,11 +194,12 @@ func TestSetNonDefault(t *testing.T) { SecurityContext: &securityContext, }, Job: &JobSpec{ - AllowNonRestoredState: &jobAllowNonRestoredState, - Parallelism: &jobParallelism, - NoLoggingToStdout: &jobNoLoggingToStdout, - RestartPolicy: &jobRestartPolicy, - SecurityContext: &securityContext, + AllowNonRestoredState: &jobAllowNonRestoredState, + Parallelism: &jobParallelism, + NoLoggingToStdout: &jobNoLoggingToStdout, + RestartPolicy: &jobRestartPolicy, + SavepointMaxAgeForUpdateSeconds: &jobSavepointMaxAgeForUpdateSeconds, + SecurityContext: &securityContext, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: "DeleteTaskManagers", AfterJobFails: "DeleteCluster", @@ -208,7 +210,8 @@ func TestSetNonDefault(t *testing.T) { HadoopConfig: &HadoopConfig{ MountPath: "/opt/flink/hadoop/conf", }, - EnvVars: nil, + EnvVars: nil, + RecreateOnUpdate: &recreateOnUpdate, }, Status: FlinkClusterStatus{}, } @@ -228,7 +231,7 @@ func TestSetNonDefault(t *testing.T) { Replicas: &jmReplicas, AccessScope: "Cluster", Ingress: &JobManagerIngressSpec{ - UseTLS: &jobManagerIngressTLSUse, + UseTLS: &jmIngressTLSUse, }, Ports: JobManagerPorts{ RPC: &jmRPCPort, @@ -257,11 +260,12 @@ func TestSetNonDefault(t *testing.T) { SecurityContext: &securityContext, }, Job: &JobSpec{ - AllowNonRestoredState: &jobAllowNonRestoredState, - Parallelism: &jobParallelism, - NoLoggingToStdout: &jobNoLoggingToStdout, - RestartPolicy: &jobRestartPolicy, - SecurityContext: &securityContext, + AllowNonRestoredState: &jobAllowNonRestoredState, + Parallelism: &jobParallelism, + NoLoggingToStdout: &jobNoLoggingToStdout, + RestartPolicy: &jobRestartPolicy, + SavepointMaxAgeForUpdateSeconds: &jobSavepointMaxAgeForUpdateSeconds, + SecurityContext: &securityContext, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: "DeleteTaskManagers", AfterJobFails: "DeleteCluster", @@ -273,7 +277,7 @@ func TestSetNonDefault(t *testing.T) { MountPath: "/opt/flink/hadoop/conf", }, EnvVars: nil, - RecreateOnUpdate: defaultRecreateOnUpdate, + RecreateOnUpdate: &recreateOnUpdate, }, Status: FlinkClusterStatus{}, } diff --git a/api/v1beta1/flinkcluster_types.go b/api/v1beta1/flinkcluster_types.go index 14c0b441..6c1b8178 100644 --- a/api/v1beta1/flinkcluster_types.go +++ b/api/v1beta1/flinkcluster_types.go @@ -98,11 +98,10 @@ const ( SavepointStateFailed = "Failed" SavepointStateSucceeded = "Succeeded" - SavepointTriggerReasonUserRequested = "user requested" - SavepointTriggerReasonScheduled = "scheduled" - SavepointTriggerReasonScheduledInitial = "scheduled initial" // The first triggered savepoint has slightly different flow - SavepointTriggerReasonJobCancel = "job cancel" - SavepointTriggerReasonUpdate = "update" + SavepointTriggerReasonUserRequested = "user requested" + SavepointTriggerReasonScheduled = "scheduled" + SavepointTriggerReasonJobCancel = "job cancel" + SavepointTriggerReasonUpdate = "update" ) // ImageSpec defines Flink image of JobManager and TaskManager containers. @@ -348,12 +347,15 @@ type JobSpec struct { // Allow non-restored state, default: false. AllowNonRestoredState *bool `json:"allowNonRestoredState,omitempty"` - // Should take savepoint before upgrading the job, default: false. - TakeSavepointOnUpgrade *bool `json:"takeSavepointOnUpgrade,omitempty"` + // Should take savepoint before updating the job, default: true. + TakeSavepointOnUpdate *bool `json:"takeSavepointOnUpdate,omitempty"` // Savepoints dir where to store savepoints of the job. SavepointsDir *string `json:"savepointsDir,omitempty"` + // Max age of savepoint allowed to progress update. + SavepointMaxAgeForUpdateSeconds *int32 `json:"savepointMaxAgeForUpdateSeconds,omitempty"` + // Automatically take a savepoint to the `savepointsDir` every n seconds. AutoSavepointSeconds *int32 `json:"autoSavepointSeconds,omitempty"` @@ -574,13 +576,12 @@ type JobStatus struct { // Last savepoint trigger ID. LastSavepointTriggerID string `json:"lastSavepointTriggerID,omitempty"` - // Last savepoint trigger time. This is updated to make sure multiple - // savepoints will not be taken simultaneously. - LastSavepointTriggerTime string `json:"lastSavepointTriggerTime,omitempty"` - // Last successful or failed savepoint operation timestamp. LastSavepointTime string `json:"lastSavepointTime,omitempty"` + // The Flink job started timestamp. + StartTime string `json:"startTime,omitempty"` + // The number of restarts. RestartCount int32 `json:"restartCount,omitempty"` } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index dc36e1da..62d3229f 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -494,6 +494,13 @@ func (in *JobManagerSpec) DeepCopyInto(out *JobManagerSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.VolumeClaimTemplates != nil { + in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates + *out = make([]v1.PersistentVolumeClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.InitContainers != nil { in, out := &in.InitContainers, &out.InitContainers *out = make([]v1.Container, len(*in)) @@ -576,8 +583,8 @@ func (in *JobSpec) DeepCopyInto(out *JobSpec) { *out = new(bool) **out = **in } - if in.TakeSavepointOnUpgrade != nil { - in, out := &in.TakeSavepointOnUpgrade, &out.takeSavepointOnUpgrade + if in.TakeSavepointOnUpdate != nil { + in, out := &in.TakeSavepointOnUpdate, &out.TakeSavepointOnUpdate *out = new(bool) **out = **in } @@ -586,6 +593,11 @@ func (in *JobSpec) DeepCopyInto(out *JobSpec) { *out = new(string) **out = **in } + if in.SavepointMaxAgeForUpdateSeconds != nil { + in, out := &in.SavepointMaxAgeForUpdateSeconds, &out.SavepointMaxAgeForUpdateSeconds + *out = new(int32) + **out = **in + } if in.AutoSavepointSeconds != nil { in, out := &in.AutoSavepointSeconds, &out.AutoSavepointSeconds *out = new(int32) @@ -774,6 +786,13 @@ func (in *TaskManagerSpec) DeepCopyInto(out *TaskManagerSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.VolumeClaimTemplates != nil { + in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates + *out = make([]v1.PersistentVolumeClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.InitContainers != nil { in, out := &in.InitContainers, &out.InitContainers *out = make([]v1.Container, len(*in)) diff --git a/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml b/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml index 9a5e5031..4b9770cb 100644 --- a/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml +++ b/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.2.4 creationTimestamp: null name: flinkclusters.flinkoperator.k8s.io spec: @@ -85,11 +85,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -163,8 +159,6 @@ spec: type: integer cancelRequested: type: boolean - takeSavepointOnUpgrade: - type: boolean className: type: string cleanupPolicy: @@ -223,11 +217,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -463,10 +453,6 @@ spec: - containerPort type: object type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -536,19 +522,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -733,19 +711,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object restartPolicy: @@ -753,6 +723,9 @@ spec: savepointGeneration: format: int32 type: integer + savepointMaxAgeForUpdateSeconds: + format: int32 + type: integer savepointsDir: type: string securityContext: @@ -808,6 +781,8 @@ spec: type: string type: object type: object + takeSavepointOnUpdate: + type: boolean volumeMounts: items: properties: @@ -985,11 +960,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -1005,11 +976,7 @@ spec: medium: type: string sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object fc: properties: @@ -1234,11 +1201,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -1507,11 +1470,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -1747,10 +1706,6 @@ spec: - containerPort type: object type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -1820,19 +1775,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -1999,11 +1946,7 @@ spec: type: object type: array memoryOffHeapMin: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string memoryOffHeapRatio: format: int32 type: integer @@ -2041,19 +1984,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -2154,11 +2089,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -2394,10 +2325,6 @@ spec: - containerPort type: object type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -2467,19 +2394,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -2661,6 +2580,111 @@ spec: type: string type: object type: array + volumeClaimTemplates: + items: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + type: array volumeMounts: items: properties: @@ -2838,11 +2862,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -2858,11 +2878,7 @@ spec: medium: type: string sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object fc: properties: @@ -3087,11 +3103,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -3355,11 +3367,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -3595,10 +3603,6 @@ spec: - containerPort type: object type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -3668,19 +3672,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -3847,11 +3843,7 @@ spec: type: object type: array memoryOffHeapMin: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string memoryOffHeapRatio: format: int32 type: integer @@ -3886,19 +3878,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -3999,11 +3983,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -4239,10 +4219,6 @@ spec: - containerPort type: object type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -4312,19 +4288,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -4506,6 +4474,111 @@ spec: type: string type: object type: array + volumeClaimTemplates: + items: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + status: + properties: + accessModes: + items: + type: string + type: array + capacity: + additionalProperties: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + type: array volumeMounts: items: properties: @@ -4683,11 +4756,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -4703,11 +4772,7 @@ spec: medium: type: string sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object fc: properties: @@ -4932,11 +4997,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -5150,8 +5211,6 @@ spec: type: string id: type: string - lastSavepointTriggerTime: - type: string lastSavepointTime: type: string lastSavepointTriggerID: @@ -5166,19 +5225,11 @@ spec: type: integer savepointLocation: type: string - state: - type: string - required: - - state - type: object - jobManagerStatefulSet: - properties: - name: + startTime: type: string state: type: string required: - - name - state type: object jobManagerIngress: @@ -5208,6 +5259,16 @@ spec: - name - state type: object + jobManagerStatefulSet: + properties: + name: + type: string + state: + type: string + required: + - name + - state + type: object taskManagerStatefulSet: properties: name: @@ -5220,8 +5281,8 @@ spec: type: object required: - configMap - - jobManagerStatefulSet - jobManagerService + - jobManagerStatefulSet - taskManagerStatefulSet type: object control: diff --git a/controllers/batchscheduler/volcano/volcano_test.go b/controllers/batchscheduler/volcano/volcano_test.go index 485f374e..099b7c97 100644 --- a/controllers/batchscheduler/volcano/volcano_test.go +++ b/controllers/batchscheduler/volcano/volcano_test.go @@ -43,7 +43,7 @@ func TestGetClusterResource(t *testing.T) { }, }, Spec: appsv1.StatefulSetSpec{ - Replicas: &jmRep, + Replicas: &jmRep, ServiceName: "flinkjobcluster-sample-jobmanager", Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -173,8 +173,8 @@ func TestGetClusterResource(t *testing.T) { }, }, Spec: appsv1.StatefulSetSpec{ - Replicas: &replicas, - ServiceName: "flinkjobcluster-sample-taskmanager", + Replicas: &replicas, + ServiceName: "flinkjobcluster-sample-taskmanager", PodManagementPolicy: "Parallel", Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ diff --git a/controllers/flinkcluster_converter.go b/controllers/flinkcluster_converter.go index a79aa403..1fd8d8c4 100644 --- a/controllers/flinkcluster_converter.go +++ b/controllers/flinkcluster_converter.go @@ -68,12 +68,12 @@ func getDesiredClusterState( return model.DesiredClusterState{} } return model.DesiredClusterState{ - ConfigMap: getDesiredConfigMap(cluster), + ConfigMap: getDesiredConfigMap(cluster), JmStatefulSet: getDesiredJobManagerStatefulSet(cluster), - JmService: getDesiredJobManagerService(cluster), - JmIngress: getDesiredJobManagerIngress(cluster), + JmService: getDesiredJobManagerService(cluster), + JmIngress: getDesiredJobManagerIngress(cluster), TmStatefulSet: getDesiredTaskManagerStatefulSet(cluster), - Job: getDesiredJob(observed), + Job: getDesiredJob(observed), } } @@ -217,9 +217,9 @@ func getDesiredJobManagerStatefulSet( Labels: statefulSetLabels, }, Spec: appsv1.StatefulSetSpec{ - Replicas: jobManagerSpec.Replicas, - Selector: &metav1.LabelSelector{MatchLabels: podLabels}, - ServiceName: jobManagerStatefulSetName, + Replicas: jobManagerSpec.Replicas, + Selector: &metav1.LabelSelector{MatchLabels: podLabels}, + ServiceName: jobManagerStatefulSetName, VolumeClaimTemplates: jobManagerSpec.VolumeClaimTemplates, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -516,11 +516,11 @@ func getDesiredTaskManagerStatefulSet( Labels: statefulSetLabels, }, Spec: appsv1.StatefulSetSpec{ - Replicas: &taskManagerSpec.Replicas, - Selector: &metav1.LabelSelector{MatchLabels: podLabels}, - ServiceName: taskManagerStatefulSetName, + Replicas: &taskManagerSpec.Replicas, + Selector: &metav1.LabelSelector{MatchLabels: podLabels}, + ServiceName: taskManagerStatefulSetName, VolumeClaimTemplates: taskManagerSpec.VolumeClaimTemplates, - PodManagementPolicy: "Parallel", + PodManagementPolicy: "Parallel", Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: podLabels, diff --git a/controllers/flinkcluster_converter_test.go b/controllers/flinkcluster_converter_test.go index 78923e62..4bcae053 100644 --- a/controllers/flinkcluster_converter_test.go +++ b/controllers/flinkcluster_converter_test.go @@ -574,8 +574,8 @@ func TestGetDesiredClusterState(t *testing.T) { }, }, Spec: appsv1.StatefulSetSpec{ - Replicas: &replicas, - ServiceName: "flinkjobcluster-sample-taskmanager", + Replicas: &replicas, + ServiceName: "flinkjobcluster-sample-taskmanager", PodManagementPolicy: "Parallel", Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ diff --git a/controllers/flinkcluster_observer.go b/controllers/flinkcluster_observer.go index 31c08fa8..b6abf1c1 100644 --- a/controllers/flinkcluster_observer.go +++ b/controllers/flinkcluster_observer.go @@ -51,17 +51,16 @@ type ObservedClusterState struct { cluster *v1beta1.FlinkCluster revisions []*appsv1.ControllerRevision configMap *corev1.ConfigMap - jmStatefulSet *appsv1.StatefulSet + jmStatefulSet *appsv1.StatefulSet jmService *corev1.Service jmIngress *extensionsv1beta1.Ingress - tmStatefulSet *appsv1.StatefulSet + tmStatefulSet *appsv1.StatefulSet job *batchv1.Job jobPod *corev1.Pod flinkJobStatus FlinkJobStatus flinkJobSubmitLog *FlinkJobSubmitLog - savepoint *flinkclient.SavepointStatus + savepoint *Savepoint revisionStatus *RevisionStatus - savepointErr error observeTime time.Time } @@ -76,6 +75,11 @@ type FlinkJobSubmitLog struct { Message string `yaml:"message"` } +type Savepoint struct { + *flinkclient.SavepointStatus + savepointErr error +} + // Observes the state of the cluster and its components. // NOT_FOUND error is ignored because it is normal, other errors are returned. func (observer *ClusterStateObserver) observe( @@ -364,12 +368,12 @@ func (observer *ClusterStateObserver) observeSavepoint(observed *ObservedCluster var err error savepoint, err = observer.flinkClient.GetSavepointStatus(flinkAPIBaseURL, jobID, triggerID) - observed.savepoint = &savepoint + observed.savepoint = &Savepoint{SavepointStatus: &savepoint} if err == nil && len(savepoint.FailureCause.StackTrace) > 0 { err = fmt.Errorf("%s", savepoint.FailureCause.StackTrace) } if err != nil { - observed.savepointErr = err + observed.savepoint.savepointErr = err log.Info("Failed to get savepoint.", "error", err, "jobID", jobID, "triggerID", triggerID) } return err diff --git a/controllers/flinkcluster_reconciler.go b/controllers/flinkcluster_reconciler.go index 7760ca85..8ff6732b 100644 --- a/controllers/flinkcluster_reconciler.go +++ b/controllers/flinkcluster_reconciler.go @@ -479,7 +479,7 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { } // Create Flink job submitter - log.Info("Updating job status to create new job submitter") + log.Info("Updating job status to proceed creating new job submitter") err = reconciler.updateStatusForNewJob() if err != nil { log.Info("Not proceed to create new job submitter because job status update failed") @@ -503,27 +503,34 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { var jobID = reconciler.getFlinkJobID() var restartPolicy = observed.cluster.Spec.Job.RestartPolicy var recordedJobStatus = observed.cluster.Status.Components.Job - var jobSpec = reconciler.observed.cluster.Spec.Job - // Update or recover Flink job by restart. + // Stop Flink job for update or recovery. + var stopReason string if shouldUpdateJob(observed) { - log.Info("Job is about to be restarted to update") - err := reconciler.restartJob(*jobSpec.TakeSavepointOnUpgrade) - return requeueResult, err + // Change job state. + err := reconciler.changeJobStateToUpdating() + if err != nil { + log.Error(err, "Failed to change job status for update", "error", err) + return requeueResult, err + } + stopReason = "update" } else if shouldRestartJob(restartPolicy, recordedJobStatus) { - log.Info("Job is about to be restarted to recover failure") - err := reconciler.restartJob(false) + stopReason = "recovery" + } + if stopReason != "" { + log.Info(fmt.Sprintf("Restart job for %s.", stopReason)) + err := reconciler.restartJob() + if err != nil { + log.Info("Failed to restart job.") + } return requeueResult, err } // Trigger savepoint if required. if len(jobID) > 0 { - shouldTakeSavepont, savepointTriggerReason := reconciler.shouldTakeSavepoint() - if shouldTakeSavepont { - err = reconciler.updateSavepointTriggerTimeStatus() - if err == nil { - newSavepointStatus, _ = reconciler.takeSavepointAsync(jobID, savepointTriggerReason) - } + shouldTakeSavepoint, savepointTriggerReason := reconciler.shouldTakeSavepoint() + if shouldTakeSavepoint { + newSavepointStatus, _ = reconciler.takeSavepointAsync(jobID, savepointTriggerReason) } } log.Info("Job is not finished yet, no action", "jobID", jobID) @@ -608,15 +615,13 @@ func (reconciler *ClusterReconciler) getFlinkJobID() string { return "" } -func (reconciler *ClusterReconciler) restartJob(shouldTakeSavepoint bool) error { +func (reconciler *ClusterReconciler) restartJob() error { var log = reconciler.log var observedJob = reconciler.observed.job var observedFlinkJob = reconciler.observed.flinkJobStatus.flinkJob log.Info("Stopping Flink job to restart", "", observedFlinkJob) - shouldTakeSavepoint = shouldTakeSavepoint && canTakeSavepoint(*reconciler.observed.cluster) - - var err = reconciler.cancelRunningJobs(shouldTakeSavepoint /* takeSavepoint */) + var err = reconciler.cancelRunningJobs(false /* takeSavepoint */) if err != nil { return err } @@ -625,7 +630,7 @@ func (reconciler *ClusterReconciler) restartJob(shouldTakeSavepoint bool) error var err = reconciler.deleteJob(observedJob) if err != nil { log.Error( - err, "Failed to delete failed job", "job", observedJob) + err, "Failed to delete job submitter", "job", observedJob) return err } } @@ -646,12 +651,17 @@ func (reconciler *ClusterReconciler) cancelUnexpectedJobs( // Cancel running jobs. func (reconciler *ClusterReconciler) cancelRunningJobs( takeSavepoint bool) error { + var log = reconciler.log var runningJobs = reconciler.observed.flinkJobStatus.flinkJobsUnexpected var flinkJob = reconciler.observed.flinkJobStatus.flinkJob if flinkJob != nil && flinkJob.ID != "" && getFlinkJobDeploymentState(flinkJob.Status) == v1beta1.JobStateRunning { runningJobs = append(runningJobs, flinkJob.ID) } + if len(runningJobs) == 0 { + log.Info("No running Flink jobs to stop.") + return nil + } return reconciler.cancelJobs(takeSavepoint, runningJobs) } @@ -690,7 +700,7 @@ func (reconciler *ClusterReconciler) cancelFlinkJob(jobID string, takeSavepoint // Trigger savepoint if it is possible, then return the savepoint status to update. // When savepoint was already triggered, return the current observed status. -// If triggering savepoint is impossible or skipped or triggered savepoint was created, proceed to stop the job. +// If savepoint cannot be triggered, taking savepoint is skipped, or the triggered savepoint is completed, proceed to stop the job. func (reconciler *ClusterReconciler) cancelFlinkJobAsync(jobID string, takeSavepoint bool) (*v1beta1.SavepointStatus, error) { var log = reconciler.log var cluster = reconciler.observed.cluster @@ -778,18 +788,14 @@ func (reconciler *ClusterReconciler) shouldTakeSavepoint() (bool, string) { return false, "" } - var nextOkTriggerTime = getTimeAfterAddedSeconds(jobStatus.LastSavepointTriggerTime, SavepointTimeoutSec) - if time.Now().Before(nextOkTriggerTime) { - return false, "" - } - - // First savepoint. + // Scheduled, check if next trigger time arrived. + var compareTime string if len(jobStatus.LastSavepointTime) == 0 { - return true, v1beta1.SavepointTriggerReasonScheduledInitial + compareTime = jobStatus.StartTime + } else { + compareTime = jobStatus.LastSavepointTime } - - // Scheduled, check if next trigger time arrived. - var nextTime = getTimeAfterAddedSeconds(jobStatus.LastSavepointTime, int64(*jobSpec.AutoSavepointSeconds)) + var nextTime = getTimeAfterAddedSeconds(compareTime, int64(*jobSpec.AutoSavepointSeconds)) return time.Now().After(nextTime), v1beta1.SavepointTriggerReasonScheduled } @@ -866,14 +872,6 @@ func (reconciler *ClusterReconciler) takeSavepoint( return err } -func (reconciler *ClusterReconciler) updateSavepointTriggerTimeStatus() error { - var cluster = v1beta1.FlinkCluster{} - reconciler.observed.cluster.DeepCopyInto(&cluster) - var jobStatus = cluster.Status.Components.Job - setTimestamp(&jobStatus.LastSavepointTriggerTime) - return reconciler.k8sClient.Status().Update(reconciler.context, &cluster) -} - func (reconciler *ClusterReconciler) updateSavepointStatus( savepointStatus flinkclient.SavepointStatus) error { var cluster = v1beta1.FlinkCluster{} @@ -883,6 +881,11 @@ func (reconciler *ClusterReconciler) updateSavepointStatus( jobStatus.SavepointGeneration++ jobStatus.LastSavepointTriggerID = savepointStatus.TriggerID jobStatus.SavepointLocation = savepointStatus.Location + + // TODO: LastSavepointTime should be set with the timestamp generated in job manager. + // Currently savepoint complete timestamp is not included in savepoints API response. + // Whereas checkpoint API returns the timestamp latest_ack_timestamp. + // Note: https://ci.apache.org/projects/flink/flink-docs-stable/ops/rest_api.html#jobs-jobid-checkpoints-details-checkpointid setTimestamp(&jobStatus.LastSavepointTime) setTimestamp(&cluster.Status.LastUpdateTime) } @@ -1015,3 +1018,13 @@ func (reconciler *ClusterReconciler) updateStatusForNewJob() error { } return err } + +func (reconciler *ClusterReconciler) changeJobStateToUpdating() error { + var clusterClone = reconciler.observed.cluster.DeepCopy() + var newJobStatus = clusterClone.Status.Components.Job + newJobStatus.ID = "" + newJobStatus.State = v1beta1.JobStateUpdating + setTimestamp(&clusterClone.Status.LastUpdateTime) + err := reconciler.k8sClient.Status().Update(reconciler.context, clusterClone) + return err +} diff --git a/controllers/flinkcluster_submit_job_script.go b/controllers/flinkcluster_submit_job_script.go index 59ee9780..466d2c5c 100644 --- a/controllers/flinkcluster_submit_job_script.go +++ b/controllers/flinkcluster_submit_job_script.go @@ -52,7 +52,7 @@ function check_jm_ready() { # Waiting for 5 mins. local -r MAX_RETRY=60 local -r RETRY_INTERVAL=5s - local -r REQUIRED_SUCCESS_NUMBER=2 + local -r REQUIRED_SUCCESS_NUMBER=3 local success_count=0 echo_log "Checking job manager to be ready. Will check success of ${REQUIRED_SUCCESS_NUMBER} API calls for stable job submission." "job_check_log" diff --git a/controllers/flinkcluster_updater.go b/controllers/flinkcluster_updater.go index 49706f7a..bed50a32 100644 --- a/controllers/flinkcluster_updater.go +++ b/controllers/flinkcluster_updater.go @@ -24,6 +24,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/googlecloudplatform/flink-operator/controllers/flinkclient" "k8s.io/apimachinery/pkg/types" "reflect" "time" @@ -388,16 +389,8 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( } // (Optional) Job. - var jobStopped = false - var jobStatus = updater.getJobStatus() - status.Components.Job = jobStatus - if jobStatus != nil && - (jobStatus.State == v1beta1.JobStateSucceeded || - jobStatus.State == v1beta1.JobStateFailed || - jobStatus.State == v1beta1.JobStateCancelled || - jobStatus.State == v1beta1.JobStateSuspended) { - jobStopped = true - } + // Update job status. + status.Components.Job = updater.getJobStatus() // Derive the new cluster state. switch recorded.State { @@ -421,9 +414,10 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( } case v1beta1.ClusterStateRunning, v1beta1.ClusterStateReconciling: + var jobStatus = status.Components.Job if isClusterUpdating { status.State = v1beta1.ClusterStateUpdating - } else if jobStopped { + } else if isJobStopped(jobStatus) { var policy = observed.cluster.Spec.Job.CleanupPolicy if jobStatus.State == v1beta1.JobStateSucceeded && policy.AfterJobSucceeds != v1beta1.CleanupActionKeepCluster { @@ -463,64 +457,13 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( panic(fmt.Sprintf("Unknown cluster state: %v", recorded.State)) } - // Savepoint status - // update savepoint status if it is in progress + // Update savepoint status if it is in progress or requested. if recorded.Savepoint != nil { - var newSavepointStatus = recorded.Savepoint.DeepCopy() - if recorded.Savepoint.State == v1beta1.SavepointStateInProgress && observed.savepoint != nil { - switch { - case observed.savepoint.IsSuccessful(): - newSavepointStatus.State = v1beta1.SavepointStateSucceeded - case observed.savepoint.IsFailed(): - var msg string - newSavepointStatus.State = v1beta1.SavepointStateFailed - if observed.savepoint.FailureCause.StackTrace != "" { - msg = fmt.Sprintf("Savepoint error: %v", observed.savepoint.FailureCause.StackTrace) - } else if observed.savepointErr != nil { - msg = fmt.Sprintf("Failed to get triggered savepoint status: %v", observed.savepointErr) - } else { - msg = "Failed to get triggered savepoint status" - } - if len(msg) > 1024 { - msg = msg[:1024] + "..." - } - newSavepointStatus.Message = msg - // TODO: organize more making savepoint status - if newSavepointStatus.TriggerReason == v1beta1.SavepointTriggerReasonUpdate { - newSavepointStatus.Message = - "Failed to take savepoint for update. " + - "The update process is being postponed until a savepoint is available. " + newSavepointStatus.Message - } - } - } - if newSavepointStatus.State == v1beta1.SavepointStateNotTriggered || newSavepointStatus.State == v1beta1.SavepointStateInProgress { - var flinkJobID = updater.getFlinkJobID() - switch { - case savepointTimeout(newSavepointStatus): - newSavepointStatus.State = v1beta1.SavepointStateFailed - newSavepointStatus.Message = "Timed out taking savepoint." - case isJobStopped(recorded.Components.Job): - newSavepointStatus.Message = "Flink job is stopped." - newSavepointStatus.State = v1beta1.SavepointStateFailed - case !isFlinkAPIReady(*observed): - newSavepointStatus.Message = "Flink API is not available." - newSavepointStatus.State = v1beta1.SavepointStateFailed - case flinkJobID == nil: - newSavepointStatus.Message = "Flink job is not submitted or identified." - newSavepointStatus.State = v1beta1.SavepointStateFailed - case flinkJobID != nil && (recorded.Savepoint.TriggerID != "" && *flinkJobID != recorded.Savepoint.JobID): - newSavepointStatus.Message = "Savepoint triggered Flink job is lost." - newSavepointStatus.State = v1beta1.SavepointStateFailed - } - // TODO: organize more making savepoint status - if newSavepointStatus.State == v1beta1.SavepointStateFailed && - newSavepointStatus.TriggerReason == v1beta1.SavepointTriggerReasonUpdate { - newSavepointStatus.Message = - "Failed to take savepoint for update. " + - "The update process is being postponed until a savepoint is available. " + newSavepointStatus.Message - } - } - status.Savepoint = newSavepointStatus + status.Savepoint = updater.getSavepointStatus( + observed.savepoint, + recorded.Savepoint, + recorded.Components.Job, + updater.getFlinkJobID()) } // User requested control @@ -531,7 +474,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( if recorded.Control != nil && userControl == recorded.Control.Name && recorded.Control.State == v1beta1.ControlStateProgressing { controlStatus = recorded.Control.DeepCopy() - var savepointStatus = status.Savepoint + var newSavepointStatus = status.Savepoint switch recorded.Control.Name { case v1beta1.ControlNameJobCancel: if status.Components.Job.State == v1beta1.JobStateCancelled { @@ -541,7 +484,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( controlStatus.Message = "Aborted job cancellation: Job is terminated." controlStatus.State = v1beta1.ControlStateFailed setTimestamp(&controlStatus.UpdateTime) - } else if savepointStatus != nil && savepointStatus.State == v1beta1.SavepointStateFailed { + } else if newSavepointStatus != nil && newSavepointStatus.State == v1beta1.SavepointStateFailed { controlStatus.Message = "Aborted job cancellation: failed to create savepoint." controlStatus.State = v1beta1.ControlStateFailed setTimestamp(&controlStatus.UpdateTime) @@ -550,11 +493,11 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( setTimestamp(&controlStatus.UpdateTime) } case v1beta1.ControlNameSavepoint: - if savepointStatus != nil { - if savepointStatus.State == v1beta1.SavepointStateSucceeded { + if newSavepointStatus != nil { + if newSavepointStatus.State == v1beta1.SavepointStateSucceeded { controlStatus.State = v1beta1.ControlStateSucceeded setTimestamp(&controlStatus.UpdateTime) - } else if savepointStatus.State == v1beta1.SavepointStateFailed || savepointStatus.State == v1beta1.SavepointStateTriggerFailed { + } else if newSavepointStatus.State == v1beta1.SavepointStateFailed || newSavepointStatus.State == v1beta1.SavepointStateTriggerFailed { controlStatus.State = v1beta1.ControlStateFailed setTimestamp(&controlStatus.UpdateTime) } @@ -586,9 +529,10 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( updater.log.Info(fmt.Sprintf(v1beta1.InvalidJobStateForSavepointMsg, v1beta1.ControlAnnotation)) break } - // Clear status for new savepoint + // Although a savepoint is in progress, if user requested savepoint, new savepoint should be triggered. status.Savepoint = getRequestedSavepointStatus(v1beta1.SavepointTriggerReasonUserRequested) controlStatus = getNewUserControlStatus(userControl) + updater.log.Info("Marked to take new savepoint user requested.") case v1beta1.ControlNameJobCancel: if isJobTerminated(observed.cluster.Spec.Job.RestartPolicy, recorded.Components.Job) { updater.log.Info(fmt.Sprintf(v1beta1.InvalidJobStateForJobCancelMsg, v1beta1.ControlAnnotation)) @@ -598,10 +542,10 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( var observedSavepoint = observed.cluster.Status.Savepoint if observedSavepoint == nil || (observedSavepoint.State != v1beta1.SavepointStateInProgress && observedSavepoint.State != v1beta1.SavepointStateNotTriggered) { - updater.log.Info("There is no savepoint in progress. Trigger savepoint in reconciler.") status.Savepoint = getRequestedSavepointStatus(v1beta1.SavepointTriggerReasonJobCancel) + updater.log.Info("Marked to take new savepoint for job cancel.") } else { - updater.log.Info("There is a savepoint in progress. Skip new savepoint.") + updater.log.Info("There is a savepoint in progress. No need to trigger new savepoint for job cancel.") } controlStatus = getNewUserControlStatus(userControl) } @@ -614,36 +558,36 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( status.Control = controlStatus // Handle update. - var savepointForJobUpdate *v1beta1.SavepointStatus switch updateState { case UpdateStatePreparing: // Even if savepoint has been created for update already, we check the age of savepoint continually. // If created savepoint is old and savepoint can be triggered, we should take savepoint again. // (e.g., for the case update is not progressed by accidents like network partition) - if !isSavepointUpToDate(observed.observeTime, *jobStatus) && + var jobSpec = observed.cluster.Spec.Job + var jobStatus = status.Components.Job + if !isSavepointUpToDate(observed.observeTime, jobSpec, jobStatus) && canTakeSavepoint(*observed.cluster) && (recorded.Savepoint == nil || recorded.Savepoint.State != v1beta1.SavepointStateNotTriggered) { + // TODO: apply exponential backoff retry // If failed to take savepoint, retry after SavepointRequestRetryIntervalSec. if recorded.Savepoint != nil && !hasTimeElapsed(recorded.Savepoint.RequestTime, time.Now(), SavepointRequestRetryIntervalSec) { updater.log.Info(fmt.Sprintf("Will retry to trigger savepoint for update, in %v seconds because previous request was failed", SavepointRequestRetryIntervalSec)) } else { + // Mark to take new savepoint for update. status.Savepoint = getRequestedSavepointStatus(v1beta1.SavepointTriggerReasonUpdate) - updater.log.Info("Savepoint will be triggered for update") + updater.log.Info("Marked to take new savepoint for update.") } } else if recorded.Savepoint != nil && recorded.Savepoint.State == v1beta1.SavepointStateInProgress { - updater.log.Info("Savepoint for update is in progress") + updater.log.Info("Savepoint for update is in progress.") } else { - updater.log.Info("Stopping job for update") + updater.log.Info("Job is stopping for update.") } case UpdateStateInProgress: - updater.log.Info("Updating cluster") + updater.log.Info("Update is in progress.") case UpdateStateFinished: status.CurrentRevision = observed.cluster.Status.NextRevision - updater.log.Info("Finished update") - } - if savepointForJobUpdate != nil { - status.Savepoint = savepointForJobUpdate + updater.log.Info("Finished update.") } // Update revision status @@ -704,12 +648,11 @@ func (updater *ClusterStatusUpdater) getJobStatus() *v1beta1.JobStatus { } newJobStatus = recordedJobStatus.DeepCopy() - // Determine job state + // Derive job state var jobState string switch { - // Updating state - case isUpdateTriggered(observedCluster.Status) && - (isJobStopped(recordedJobStatus) || observedCluster.Status.State == v1beta1.ClusterStateStopped): + // When updating stopped job + case isUpdateTriggered(observedCluster.Status) && isJobStopped(recordedJobStatus): jobState = v1beta1.JobStateUpdating // Already terminated state case isJobTerminated(observedCluster.Spec.Job.RestartPolicy, recordedJobStatus): @@ -718,7 +661,7 @@ func (updater *ClusterStatusUpdater) getJobStatus() *v1beta1.JobStatus { case observedFlinkJob != nil: jobState = getFlinkJobDeploymentState(observedFlinkJob.Status) if jobState == "" { - updater.log.Error(errors.New("failed to determine Flink job deployment state"), "observed flink job status", observedFlinkJob.Status) + updater.log.Error(errors.New("failed to get Flink job deployment state"), "observed flink job status", observedFlinkJob.Status) jobState = recordedJobStatus.State } // When Flink job not found @@ -763,11 +706,22 @@ func (updater *ClusterStatusUpdater) getJobStatus() *v1beta1.JobStatus { // State newJobStatus.State = jobState + // Flink job start time + // TODO: It would be nice to set StartTime with the timestamp retrieved from the Flink job API like /jobs/{job-id}. + if jobState == v1beta1.JobStateRunning && newJobStatus.StartTime == "" { + setTimestamp(&newJobStatus.StartTime) + } + // Savepoint if newJobStatus != nil && observedSavepoint != nil && observedSavepoint.IsSuccessful() { newJobStatus.SavepointGeneration++ newJobStatus.LastSavepointTriggerID = observedSavepoint.TriggerID newJobStatus.SavepointLocation = observedSavepoint.Location + + // TODO: LastSavepointTime should be set with the timestamp generated in job manager. + // Currently savepoint complete timestamp is not included in savepoints API response. + // Whereas checkpoint API returns the timestamp latest_ack_timestamp. + // Note: https://ci.apache.org/projects/flink/flink-docs-stable/ops/rest_api.html#jobs-jobid-checkpoints-details-checkpointid setTimestamp(&newJobStatus.LastSavepointTime) } @@ -939,6 +893,54 @@ func (updater *ClusterStatusUpdater) clearControlAnnotation(newControlStatus *v1 return nil } +func (updater *ClusterStatusUpdater) getSavepointStatus( + observedSavepoint *Savepoint, + recordedSavepointStatus *v1beta1.SavepointStatus, + recordedJobStatus *v1beta1.JobStatus, + flinkJobID *string) *v1beta1.SavepointStatus { + var savepointStatus = recordedSavepointStatus.DeepCopy() + var errMsg string + if recordedSavepointStatus.State == v1beta1.SavepointStateInProgress && observedSavepoint != nil { + switch { + case observedSavepoint.IsSuccessful(): + savepointStatus.State = v1beta1.SavepointStateSucceeded + case observedSavepoint.IsFailed(): + savepointStatus.State = v1beta1.SavepointStateFailed + errMsg = fmt.Sprintf("Savepoint error: %v", observedSavepoint.FailureCause.StackTrace) + case observedSavepoint.savepointErr != nil: + if err, ok := observedSavepoint.savepointErr.(*flinkclient.HTTPError); ok { + savepointStatus.State = v1beta1.SavepointStateFailed + errMsg = fmt.Sprintf("Failed to get savepoint status: %v", err) + } + } + } + if savepointStatus.State == v1beta1.SavepointStateNotTriggered || savepointStatus.State == v1beta1.SavepointStateInProgress { + switch { + case isJobStopped(recordedJobStatus): + errMsg = "Flink job is stopped." + savepointStatus.State = v1beta1.SavepointStateFailed + case flinkJobID == nil: + errMsg = "Flink job is not identified." + savepointStatus.State = v1beta1.SavepointStateFailed + case flinkJobID != nil && (recordedSavepointStatus.TriggerID != "" && *flinkJobID != recordedSavepointStatus.JobID): + errMsg = "Savepoint triggered Flink job is lost." + savepointStatus.State = v1beta1.SavepointStateFailed + } + } + if errMsg != "" { + if savepointStatus.TriggerReason == v1beta1.SavepointTriggerReasonUpdate { + errMsg = + "Failed to take savepoint for update. " + + "The update process is being postponed until a savepoint is available. " + errMsg + } + if len(errMsg) > 1024 { + errMsg = errMsg[:1024] + } + savepointStatus.Message = errMsg + } + return savepointStatus +} + func getStatefulSetState(statefulSet *appsv1.StatefulSet) string { if statefulSet.Status.ReadyReplicas >= *statefulSet.Spec.Replicas { return v1beta1.ComponentStateReady diff --git a/controllers/flinkcluster_util.go b/controllers/flinkcluster_util.go index 28607aef..b1ebcff4 100644 --- a/controllers/flinkcluster_util.go +++ b/controllers/flinkcluster_util.go @@ -42,13 +42,10 @@ const ( ControlRetries = "retries" ControlMaxRetries = "3" - SavepointTimeoutSec = 900 // 15 mins - RevisionNameLabel = "flinkoperator.k8s.io/revision-name" - // TODO: need to be user configurable - SavepointAgeForJobUpdateSec = 300 - SavepointRequestRetryIntervalSec = 10 + SavepointMaxAgeForUpdateSecondsDefault = 300 // 5 min + SavepointRequestRetryIntervalSec = 10 ) type UpdateState string @@ -143,11 +140,12 @@ func canTakeSavepoint(cluster v1beta1.FlinkCluster) bool { var savepointStatus = cluster.Status.Savepoint var jobStatus = cluster.Status.Components.Job return jobSpec != nil && jobSpec.SavepointsDir != nil && - !isJobStopped(jobStatus) && + jobStatus.State == v1beta1.JobStateRunning && (savepointStatus == nil || savepointStatus.State != v1beta1.SavepointStateInProgress) } // shouldRestartJob returns true if the controller should restart failed or lost job. +// The controller can restart the job only if there is a savepoint to restore, recorded in status field. func shouldRestartJob( restartPolicy *v1beta1.JobRestartPolicy, jobStatus *v1beta1.JobStatus) bool { @@ -158,9 +156,17 @@ func shouldRestartJob( len(jobStatus.SavepointLocation) > 0 } +// shouldUpdateJob returns true if the controller should update the job. +// The controller should update the job when update is triggered and it is prepared to update. +// When the job is stopped, no savepoint is required, or the savepoint recorded in status field is up to date, it is ready to update. func shouldUpdateJob(observed ObservedClusterState) bool { var jobStatus = observed.cluster.Status.Components.Job - var readyToUpdate = jobStatus == nil || isJobStopped(jobStatus) || isSavepointUpToDate(observed.observeTime, *jobStatus) + var jobSpec = observed.cluster.Spec.Job + var takeSavepointOnUpdate = jobSpec.TakeSavepointOnUpdate == nil || *jobSpec.TakeSavepointOnUpdate == true + var readyToUpdate = jobStatus == nil || + isJobStopped(jobStatus) || + !takeSavepointOnUpdate || + isSavepointUpToDate(observed.observeTime, jobSpec, jobStatus) return isUpdateTriggered(observed.cluster.Status) && readyToUpdate } @@ -302,16 +308,6 @@ func getRequestedSavepointStatus(triggerReason string) *v1beta1.SavepointStatus } } -func savepointTimeout(s *v1beta1.SavepointStatus) bool { - if s.TriggerTime == "" { - return false - } - tc := &TimeConverter{} - triggerTime := tc.FromString(s.TriggerTime) - validTime := triggerTime.Add(time.Duration(int64(SavepointTimeoutSec) * int64(time.Second))) - return time.Now().After(validTime) -} - func getControlEvent(status v1beta1.FlinkClusterControlStatus) (eventType string, eventReason string, eventMessage string) { var msg = status.Message if len(msg) > 100 { @@ -402,10 +398,16 @@ func isUserControlFinished(controlStatus *v1beta1.FlinkClusterControlStatus) boo controlStatus.State == v1beta1.ControlStateFailed } -// Check if the savepoint has been created recently. -func isSavepointUpToDate(now time.Time, jobStatus v1beta1.JobStatus) bool { +// Check if the savepoint is up to date. +func isSavepointUpToDate(now time.Time, jobSpec *v1beta1.JobSpec, jobStatus *v1beta1.JobStatus) bool { if jobStatus.SavepointLocation != "" && jobStatus.LastSavepointTime != "" { - if !hasTimeElapsed(jobStatus.LastSavepointTime, now, SavepointAgeForJobUpdateSec) { + var spMaxAge int + if jobSpec.SavepointMaxAgeForUpdateSeconds != nil { + spMaxAge = int(*jobSpec.SavepointMaxAgeForUpdateSeconds) + } else { + spMaxAge = SavepointMaxAgeForUpdateSecondsDefault + } + if !hasTimeElapsed(jobStatus.LastSavepointTime, now, spMaxAge) { return true } } @@ -517,16 +519,17 @@ func isFlinkAPIReady(observed ObservedClusterState) bool { func getUpdateState(observed ObservedClusterState) UpdateState { var recordedJobStatus = observed.cluster.Status.Components.Job - if !isUpdateTriggered(observed.cluster.Status) { + + switch { + case !isUpdateTriggered(observed.cluster.Status): return "" - } - if isJobActive(recordedJobStatus) { + case isJobActive(recordedJobStatus): return UpdateStatePreparing - } - if isClusterUpdateToDate(observed) { + case isClusterUpdateToDate(observed): return UpdateStateFinished + default: + return UpdateStateInProgress } - return UpdateStateInProgress } func getNonLiveHistory(revisions []*appsv1.ControllerRevision, historyLimit int) []*appsv1.ControllerRevision { diff --git a/controllers/flinkcluster_util_test.go b/controllers/flinkcluster_util_test.go index e90671c3..47a6faf0 100644 --- a/controllers/flinkcluster_util_test.go +++ b/controllers/flinkcluster_util_test.go @@ -225,15 +225,18 @@ func TestShouldUpdateJob(t *testing.T) { var tc = &TimeConverter{} var savepointTime = time.Now() var observeTime = savepointTime.Add(time.Second * 100) + var savepointMaxAgeForUpdateSeconds = int32(300) var observed = ObservedClusterState{ observeTime: observeTime, cluster: &v1beta1.FlinkCluster{ + Spec: v1beta1.FlinkClusterSpec{Job: &v1beta1.JobSpec{ + SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, + }}, Status: v1beta1.FlinkClusterStatus{ Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{ - State: v1beta1.JobStateRunning, - LastSavepointTime: tc.ToString(savepointTime), - LastSavepointTriggerTime: tc.ToString(savepointTime), - SavepointLocation: "gs://my-bucket/savepoint-123", + State: v1beta1.JobStateRunning, + LastSavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/savepoint-123", }}, CurrentRevision: "1", NextRevision: "2", }, @@ -245,6 +248,9 @@ func TestShouldUpdateJob(t *testing.T) { // should update when update triggered and job failed. observed = ObservedClusterState{ cluster: &v1beta1.FlinkCluster{ + Spec: v1beta1.FlinkClusterSpec{Job: &v1beta1.JobSpec{ + SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, + }}, Status: v1beta1.FlinkClusterStatus{ Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{ State: v1beta1.JobStateFailed, @@ -263,12 +269,14 @@ func TestShouldUpdateJob(t *testing.T) { observed = ObservedClusterState{ observeTime: observeTime, cluster: &v1beta1.FlinkCluster{ + Spec: v1beta1.FlinkClusterSpec{Job: &v1beta1.JobSpec{ + SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, + }}, Status: v1beta1.FlinkClusterStatus{ Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{ - State: v1beta1.JobStateRunning, - LastSavepointTime: tc.ToString(savepointTime), - LastSavepointTriggerTime: tc.ToString(savepointTime), - SavepointLocation: "gs://my-bucket/savepoint-123", + State: v1beta1.JobStateRunning, + LastSavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/savepoint-123", }}, CurrentRevision: "1", NextRevision: "2", }, @@ -278,12 +286,11 @@ func TestShouldUpdateJob(t *testing.T) { assert.Equal(t, update, false) // cannot update without savepointLocation - tc = &TimeConverter{} - savepointTime = time.Now() - observeTime = savepointTime.Add(time.Second * 500) observed = ObservedClusterState{ - observeTime: observeTime, cluster: &v1beta1.FlinkCluster{ + Spec: v1beta1.FlinkClusterSpec{Job: &v1beta1.JobSpec{ + SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, + }}, Status: v1beta1.FlinkClusterStatus{ Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{ State: v1beta1.JobStateUpdating, @@ -294,6 +301,25 @@ func TestShouldUpdateJob(t *testing.T) { } update = shouldUpdateJob(observed) assert.Equal(t, update, false) + + // proceed update without savepointLocation if takeSavepoint is false. + takeSavepointOnUpdate := false + observed = ObservedClusterState{ + cluster: &v1beta1.FlinkCluster{ + Spec: v1beta1.FlinkClusterSpec{Job: &v1beta1.JobSpec{ + TakeSavepointOnUpdate: &takeSavepointOnUpdate, + SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, + }}, + Status: v1beta1.FlinkClusterStatus{ + Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{ + State: v1beta1.JobStateUpdating, + }}, + CurrentRevision: "1", NextRevision: "2", + }, + }, + } + update = shouldUpdateJob(observed) + assert.Equal(t, update, true) } func TestGetNextRevisionNumber(t *testing.T) { @@ -326,36 +352,37 @@ func TestIsSavepointUpToDate(t *testing.T) { var tc = &TimeConverter{} var savepointTime = time.Now() var observeTime = savepointTime.Add(time.Second * 100) + var savepointMaxAgeForUpdateSeconds = int32(300) + var jobSpec = v1beta1.JobSpec{ + SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, + } var jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - LastSavepointTime: tc.ToString(savepointTime), - LastSavepointTriggerTime: tc.ToString(savepointTime), - SavepointLocation: "gs://my-bucket/savepoint-123", + State: v1beta1.JobStateFailed, + LastSavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/savepoint-123", } - var update = isSavepointUpToDate(observeTime, jobStatus) + var update = isSavepointUpToDate(observeTime, &jobSpec, &jobStatus) assert.Equal(t, update, true) // old savepointTime = time.Now() observeTime = savepointTime.Add(time.Second * 500) jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - LastSavepointTime: tc.ToString(savepointTime), - LastSavepointTriggerTime: tc.ToString(savepointTime), - SavepointLocation: "gs://my-bucket/savepoint-123", + State: v1beta1.JobStateFailed, + LastSavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/savepoint-123", } - update = isSavepointUpToDate(observeTime, jobStatus) + update = isSavepointUpToDate(observeTime, &jobSpec, &jobStatus) assert.Equal(t, update, false) // Fails without savepointLocation savepointTime = time.Now() observeTime = savepointTime.Add(time.Second * 500) jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - LastSavepointTime: tc.ToString(savepointTime), - LastSavepointTriggerTime: tc.ToString(savepointTime), + State: v1beta1.JobStateFailed, + LastSavepointTime: tc.ToString(savepointTime), } - update = isSavepointUpToDate(observeTime, jobStatus) + update = isSavepointUpToDate(observeTime, &jobSpec, &jobStatus) assert.Equal(t, update, false) } diff --git a/controllers/model/model.go b/controllers/model/model.go index 9faf6d39..968b6df3 100644 --- a/controllers/model/model.go +++ b/controllers/model/model.go @@ -23,9 +23,9 @@ import ( // DesiredClusterState holds desired state of a cluster. type DesiredClusterState struct { JmStatefulSet *appsv1.StatefulSet - JmService *corev1.Service - JmIngress *extensionsv1beta1.Ingress + JmService *corev1.Service + JmIngress *extensionsv1beta1.Ingress TmStatefulSet *appsv1.StatefulSet - ConfigMap *corev1.ConfigMap - Job *batchv1.Job + ConfigMap *corev1.ConfigMap + Job *batchv1.Job } diff --git a/docs/crd.md b/docs/crd.md index 1eef7a35..ae20ff4b 100644 --- a/docs/crd.md +++ b/docs/crd.md @@ -69,7 +69,7 @@ FlinkCluster |__ args |__ fromSavepoint |__ allowNonRestoredState - |__ takeSavepointOnUpgrade + |__ takeSavepointOnUpdate |__ autoSavepointSeconds |__ savepointsDir |__ savepointGeneration @@ -263,7 +263,7 @@ FlinkCluster * **autoSavepointSeconds** (optional): Automatically take a savepoint to the `savepointsDir` every n seconds. * **savepointsDir** (optional): Savepoints dir where to store automatically taken savepoints. * **allowNonRestoredState** (optional): Allow non-restored state, default: false. - * **takeSavepointOnUpgrade** (optional): Should take savepoint before upgrading the job, default: false. + * **takeSavepointOnUpdate** (optional): Should take savepoint before updating the job, default: false. * **savepointGeneration** (optional): Update this field to `jobStatus.savepointGeneration + 1` for a running job cluster to trigger a new savepoint to `savepointsDir` on demand. * **parallelism** (optional): Parallelism of the job, default: 1. diff --git a/helm-chart/flink-operator/templates/flink-cluster-crd.yaml b/helm-chart/flink-operator/templates/flink-cluster-crd.yaml index 56a88b89..8f893723 100644 --- a/helm-chart/flink-operator/templates/flink-cluster-crd.yaml +++ b/helm-chart/flink-operator/templates/flink-cluster-crd.yaml @@ -164,7 +164,7 @@ spec: type: integer cancelRequested: type: boolean - takeSavepointOnUpgrade: + takeSavepointOnUpdate: type: boolean className: type: string @@ -4978,8 +4978,6 @@ spec: type: string lastSavepointTime: type: string - lastSavepointTriggerTime: - type: string lastSavepointTriggerID: type: string name: From 21903045e5269bb82b10de1d2ecb8ea7c9bbd6d9 Mon Sep 17 00:00:00 2001 From: elanv Date: Wed, 24 Feb 2021 01:53:19 +0900 Subject: [PATCH 2/6] Organize savepoint handling and related routines --- api/v1beta1/flinkcluster_types.go | 10 +- api/v1beta1/flinkcluster_validate.go | 2 +- api/v1beta1/flinkcluster_validate_test.go | 4 +- .../flinkoperator.k8s.io_flinkclusters.yaml | 256 ++++++++++++--- controllers/flinkcluster_reconciler.go | 304 +++++++----------- controllers/flinkcluster_updater.go | 207 +++++------- controllers/flinkcluster_util.go | 54 ++-- 7 files changed, 455 insertions(+), 382 deletions(-) diff --git a/api/v1beta1/flinkcluster_types.go b/api/v1beta1/flinkcluster_types.go index 6c1b8178..230d7793 100644 --- a/api/v1beta1/flinkcluster_types.go +++ b/api/v1beta1/flinkcluster_types.go @@ -85,14 +85,14 @@ const ( ControlNameJobCancel = "job-cancel" // control state - ControlStateProgressing = "Progressing" - ControlStateSucceeded = "Succeeded" - ControlStateFailed = "Failed" + ControlStateRequested = "Requested" + ControlStateInProgress = "InProgress" + ControlStateSucceeded = "Succeeded" + ControlStateFailed = "Failed" ) // Savepoint status const ( - SavepointStateNotTriggered = "NotTriggered" SavepointStateInProgress = "InProgress" SavepointStateTriggerFailed = "TriggerFailed" SavepointStateFailed = "Failed" @@ -601,7 +601,7 @@ type SavepointStatus struct { TriggerReason string `json:"triggerReason,omitempty"` // Savepoint requested time. - RequestTime string `json:"requestTime,omitempty"` + UpdateTime string `json:"requestTime,omitempty"` // Savepoint state. State string `json:"state"` diff --git a/api/v1beta1/flinkcluster_validate.go b/api/v1beta1/flinkcluster_validate.go index df52d8fa..949154a3 100644 --- a/api/v1beta1/flinkcluster_validate.go +++ b/api/v1beta1/flinkcluster_validate.go @@ -115,7 +115,7 @@ func (v *Validator) checkControlAnnotations(old *FlinkCluster, new *FlinkCluster oldUserControl, _ := old.Annotations[ControlAnnotation] newUserControl, ok := new.Annotations[ControlAnnotation] if ok { - if oldUserControl != newUserControl && old.Status.Control != nil && old.Status.Control.State == ControlStateProgressing { + if oldUserControl != newUserControl && old.Status.Control != nil && old.Status.Control.State == ControlStateInProgress { return fmt.Errorf(ControlChangeWarnMsg, ControlAnnotation) } switch newUserControl { diff --git a/api/v1beta1/flinkcluster_validate_test.go b/api/v1beta1/flinkcluster_validate_test.go index e1031a5a..4319a066 100644 --- a/api/v1beta1/flinkcluster_validate_test.go +++ b/api/v1beta1/flinkcluster_validate_test.go @@ -813,7 +813,7 @@ func TestUserControlSavepoint(t *testing.T) { var oldCluster1 = FlinkCluster{ Spec: FlinkClusterSpec{Job: &JobSpec{}}, - Status: FlinkClusterStatus{Control: &FlinkClusterControlStatus{State: ControlStateProgressing}}, + Status: FlinkClusterStatus{Control: &FlinkClusterControlStatus{State: ControlStateInProgress}}, } var err1 = validator.ValidateUpdate(&oldCluster1, &newCluster) var expectedErr1 = "change is not allowed for control in progress, annotation: flinkclusters.flinkoperator.k8s.io/user-control" @@ -864,7 +864,7 @@ func TestUserControlJobCancel(t *testing.T) { var oldCluster1 = FlinkCluster{ Spec: FlinkClusterSpec{Job: &JobSpec{}}, - Status: FlinkClusterStatus{Control: &FlinkClusterControlStatus{State: ControlStateProgressing}}, + Status: FlinkClusterStatus{Control: &FlinkClusterControlStatus{State: ControlStateInProgress}}, } var err1 = validator.ValidateUpdate(&oldCluster1, &newCluster) var expectedErr1 = "change is not allowed for control in progress, annotation: flinkclusters.flinkoperator.k8s.io/user-control" diff --git a/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml b/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml index 4b9770cb..8b6f61be 100644 --- a/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml +++ b/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.4 + controller-gen.kubebuilder.io/version: v0.3.0 creationTimestamp: null name: flinkclusters.flinkoperator.k8s.io spec: @@ -85,7 +85,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -217,7 +221,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -453,6 +461,10 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -522,11 +534,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -711,11 +731,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object restartPolicy: @@ -960,7 +988,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -976,7 +1008,11 @@ spec: medium: type: string sizeLimit: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object fc: properties: @@ -1201,7 +1237,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -1470,7 +1510,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -1706,6 +1750,10 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -1775,11 +1823,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -1946,7 +2002,11 @@ spec: type: object type: array memoryOffHeapMin: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true memoryOffHeapRatio: format: int32 type: integer @@ -1984,11 +2044,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -2089,7 +2157,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -2325,6 +2397,10 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -2394,11 +2470,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -2611,11 +2695,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object selector: @@ -2656,7 +2748,11 @@ spec: type: array capacity: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object conditions: items: @@ -2862,7 +2958,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -2878,7 +2978,11 @@ spec: medium: type: string sizeLimit: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object fc: properties: @@ -3103,7 +3207,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -3367,7 +3475,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -3603,6 +3715,10 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -3672,11 +3788,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -3843,7 +3967,11 @@ spec: type: object type: array memoryOffHeapMin: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true memoryOffHeapRatio: format: int32 type: integer @@ -3878,11 +4006,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -3983,7 +4119,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -4219,6 +4359,10 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -4288,11 +4432,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -4505,11 +4657,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object selector: @@ -4550,7 +4710,11 @@ spec: type: array capacity: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object conditions: items: @@ -4756,7 +4920,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -4772,7 +4940,11 @@ spec: medium: type: string sizeLimit: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object fc: properties: @@ -4997,7 +5169,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: diff --git a/controllers/flinkcluster_reconciler.go b/controllers/flinkcluster_reconciler.go index 8ff6732b..7763b04b 100644 --- a/controllers/flinkcluster_reconciler.go +++ b/controllers/flinkcluster_reconciler.go @@ -24,7 +24,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" - "reflect" "time" "github.com/go-logr/logr" @@ -451,6 +450,7 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { var recordedJobStatus = observed.cluster.Status.Components.Job var activeFlinkJob bool var err error + var jobID = reconciler.getFlinkJobID() // Update status changed via job reconciliation. var newSavepointStatus *v1beta1.SavepointStatus @@ -467,6 +467,17 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { // Check if Flink job is active if isJobActive(recordedJobStatus) { activeFlinkJob = true + + // Trigger savepoint if required. + if len(jobID) > 0 { + savepointTriggerReason := reconciler.shouldTakeSavepoint() + if savepointTriggerReason != "" { + newSavepointStatus, _ = reconciler.takeSavepointAsync(jobID, savepointTriggerReason) + if userControl := getNewUserControlRequest(observed.cluster); userControl == v1beta1.ControlNameSavepoint { + newControlStatus = getUserControlStatus(userControl, v1beta1.ControlStateInProgress) + } + } + } } else { activeFlinkJob = false } @@ -500,20 +511,19 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { } if desiredJob != nil && activeFlinkJob { - var jobID = reconciler.getFlinkJobID() var restartPolicy = observed.cluster.Spec.Job.RestartPolicy var recordedJobStatus = observed.cluster.Status.Components.Job // Stop Flink job for update or recovery. var stopReason string if shouldUpdateJob(observed) { + stopReason = "update" // Change job state. err := reconciler.changeJobStateToUpdating() if err != nil { log.Error(err, "Failed to change job status for update", "error", err) return requeueResult, err } - stopReason = "update" } else if shouldRestartJob(restartPolicy, recordedJobStatus) { stopReason = "recovery" } @@ -526,44 +536,37 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { return requeueResult, err } - // Trigger savepoint if required. - if len(jobID) > 0 { - shouldTakeSavepoint, savepointTriggerReason := reconciler.shouldTakeSavepoint() - if shouldTakeSavepoint { - newSavepointStatus, _ = reconciler.takeSavepointAsync(jobID, savepointTriggerReason) - } - } log.Info("Job is not finished yet, no action", "jobID", jobID) return requeueResult, nil } - // Stop Flink job + // Job cancel requested. Stop Flink job. if desiredJob == nil && activeFlinkJob { - // Cancel Flink job if it is live - // case 1) In the case of which savepoint was triggered, after it is completed, proceed to delete step. - // case 2) When savepoint was skipped, continue to delete step immediately. - // - // If savepoint or cancellation was failed, the control state is fallen to the failed in the updater. - var jobID = reconciler.getFlinkJobID() - log.Info("Cancelling job", "jobID", jobID) - - var savepointStatus, err = reconciler.cancelFlinkJobAsync(jobID, true /* takeSavepoint */) - if !reflect.DeepEqual(savepointStatus, observed.cluster.Status.Savepoint) { - newSavepointStatus = savepointStatus + log.Info("Stopping job", "jobID", jobID) + log.Info("Check savepoint status first.") + var savepoint *v1beta1.SavepointStatus + if newSavepointStatus != nil { + savepoint = newSavepointStatus + } else { + savepoint = observed.cluster.Status.Savepoint } + // If savepoint is taken, proceed to cancel job. + var err = reconciler.cancelFlinkJobAsync(jobID, savepoint) + // When there is an error, get failed status to record. if err != nil { log.Error(err, "Failed to cancel job", "jobID", jobID) newControlStatus = getFailedCancelStatus(err) return requeueResult, err } - // To proceed to delete step: - // case 1) savepoint triggered: savepointStatus state should be SavepointStateSucceeded and there is no error - // case 2) savepoint skipped: savepointStatus is nil and there is no error - if savepointStatus != nil && savepointStatus.State != v1beta1.SavepointStateSucceeded { + if userControl := getNewUserControlRequest(observed.cluster); userControl == v1beta1.ControlNameJobCancel { + newControlStatus = getUserControlStatus(userControl, v1beta1.ControlStateInProgress) + } + // When savepoint is in progress yet, check it in next iteration. + if savepoint != nil && savepoint.State == v1beta1.SavepointStateInProgress { return requeueResult, nil } - - return ctrl.Result{}, err + log.Info("Successfully job cancelled.") + return requeueResult, nil } if isJobStopped(recordedJobStatus) { @@ -701,113 +704,94 @@ func (reconciler *ClusterReconciler) cancelFlinkJob(jobID string, takeSavepoint // Trigger savepoint if it is possible, then return the savepoint status to update. // When savepoint was already triggered, return the current observed status. // If savepoint cannot be triggered, taking savepoint is skipped, or the triggered savepoint is completed, proceed to stop the job. -func (reconciler *ClusterReconciler) cancelFlinkJobAsync(jobID string, takeSavepoint bool) (*v1beta1.SavepointStatus, error) { +func (reconciler *ClusterReconciler) cancelFlinkJobAsync(jobID string, savepoint *v1beta1.SavepointStatus) error { var log = reconciler.log var cluster = reconciler.observed.cluster - var observedSavepoint = reconciler.observed.cluster.Status.Savepoint - var savepointStatus *v1beta1.SavepointStatus var err error - switch observedSavepoint.State { - case v1beta1.SavepointStateNotTriggered: - if takeSavepoint && canTakeSavepoint(*reconciler.observed.cluster) { - savepointStatus, err = reconciler.takeSavepointAsync(jobID, v1beta1.SavepointTriggerReasonJobCancel) - if err != nil { - log.Info("Failed to trigger savepoint.") - return savepointStatus, fmt.Errorf("failed to trigger savepoint: %v", err) - } - log.Info("Triggered savepoint and wait it is completed.") - return savepointStatus, nil - } else { - savepointStatus = nil - if takeSavepoint { - log.Info("Savepoint was desired but couldn't be triggered. Skip taking savepoint before stopping job", "jobID", jobID) - } else { - log.Info("Skip taking savepoint before stopping job", "jobID", jobID) - } - } - case v1beta1.SavepointStateInProgress: - log.Info("Triggered savepoint already and wait until it is completed.") - return observedSavepoint, nil + switch savepoint.State { case v1beta1.SavepointStateSucceeded: - savepointStatus = observedSavepoint log.Info("Successfully savepoint created. Proceed to stop job.") - // Cannot be reached here with these states, because job-cancel control should be finished with failed savepoint states by updater. + case v1beta1.SavepointStateInProgress: + log.Info("Triggered savepoint already, wait until it is completed.") + return nil case v1beta1.SavepointStateTriggerFailed: - fallthrough + log.Info("Failed to trigger savepoint.") + return fmt.Errorf("failed to trigger savepoint: %v", *savepoint) + // Cannot be reached here with these states, because job-cancel control should be finished with failed savepoint states in updater. case v1beta1.SavepointStateFailed: fallthrough default: - return nil, fmt.Errorf("unexpected savepoint status: %v", *observedSavepoint) + return fmt.Errorf("unexpected savepoint status: %v", *savepoint) } var apiBaseURL = getFlinkAPIBaseURL(cluster) log.Info("Stopping job", "jobID", jobID) err = reconciler.flinkClient.StopJob(apiBaseURL, jobID) if err != nil { - return savepointStatus, fmt.Errorf("failed to stop job: %v", err) + return fmt.Errorf("failed to stop job: %v", err) } - return savepointStatus, nil + return nil } -func (reconciler *ClusterReconciler) shouldTakeSavepoint() (bool, string) { +func (reconciler *ClusterReconciler) shouldTakeSavepoint() string { var log = reconciler.log - var jobSpec = reconciler.observed.cluster.Spec.Job - var jobStatus = reconciler.observed.cluster.Status.Components.Job - var savepointStatus = reconciler.observed.cluster.Status.Savepoint + var observed = reconciler.observed + var cluster = observed.cluster + var jobSpec = observed.cluster.Spec.Job + var jobStatus = observed.cluster.Status.Components.Job + var savepointStatus = observed.cluster.Status.Savepoint + var userControl = getNewUserControlRequest(cluster) if !canTakeSavepoint(*reconciler.observed.cluster) { - return false, "" - } - - // User requested. - - // In the case of which savepoint status is in finished state, - // savepoint trigger by spec.job.savepointGeneration is not possible - // because the field cannot be increased more when savepoint is failed. - // - // Savepoint retry by annotation is possible because the annotations would be cleared - // when the last savepoint was finished and user can attach the annotation again. - - // Savepoint can be triggered in updater for user request, job-cancel and job update - if savepointStatus != nil && savepointStatus.State == v1beta1.SavepointStateNotTriggered { - return true, savepointStatus.TriggerReason - } - + return "" + } + + switch { + // Triggered by user requested savepoint + case userControl == v1beta1.ControlNameSavepoint: + return v1beta1.SavepointTriggerReasonUserRequested + // Triggered by user requested job cancel + case userControl == v1beta1.ControlNameJobCancel: + return v1beta1.SavepointTriggerReasonJobCancel + // Triggered by update + case getUpdateState(observed) == UpdateStatePreparing: + // TODO: apply exponential backoff retry + // If failed to take savepoint, retry after SavepointRequestRetryIntervalSec. + var takeSavepointOnUpdate = jobSpec.TakeSavepointOnUpdate == nil || *jobSpec.TakeSavepointOnUpdate == true + if takeSavepointOnUpdate && + !isSavepointUpToDate(observed.observeTime, jobSpec, jobStatus) && // savepoint up to date + (savepointStatus == nil || savepointStatus.State != v1beta1.SavepointStateInProgress) && // no savepoint in progress + hasTimeElapsed(savepointStatus.UpdateTime, time.Now(), SavepointRequestRetryIntervalSec) { // retry interval arrived + return v1beta1.SavepointTriggerReasonUpdate + } + // Triggered by schedule (auto savepoint) + case jobSpec.AutoSavepointSeconds != nil: + // Check if next trigger time arrived. + var compareTime string + if len(jobStatus.LastSavepointTime) == 0 { + compareTime = jobStatus.StartTime + } else { + compareTime = jobStatus.LastSavepointTime + } + var nextTime = getTimeAfterAddedSeconds(compareTime, int64(*jobSpec.AutoSavepointSeconds)) + if time.Now().After(nextTime) { + return v1beta1.SavepointTriggerReasonScheduled + } // TODO: spec.job.savepointGeneration will be deprecated - if jobSpec.SavepointGeneration > jobStatus.SavepointGeneration && - (savepointStatus != nil && savepointStatus.State != v1beta1.SavepointStateFailed && savepointStatus.State != v1beta1.SavepointStateTriggerFailed) { + // Triggered by savepointGeneration increased + // When previous savepoint is failed, savepoint trigger by spec.job.savepointGeneration is not possible + // because the field cannot be increased more by validator. + // Note: checkSavepointGeneration in flinkcluster_validate.go + case jobSpec.SavepointGeneration > jobStatus.SavepointGeneration: log.Info( "Savepoint is requested", "statusGen", jobStatus.SavepointGeneration, "specGen", jobSpec.SavepointGeneration) - return true, v1beta1.SavepointTriggerReasonUserRequested - } - - if jobSpec.AutoSavepointSeconds == nil { - return false, "" - } - - // Scheduled, check if next trigger time arrived. - var compareTime string - if len(jobStatus.LastSavepointTime) == 0 { - compareTime = jobStatus.StartTime - } else { - compareTime = jobStatus.LastSavepointTime + return v1beta1.SavepointTriggerReasonUserRequested } - var nextTime = getTimeAfterAddedSeconds(compareTime, int64(*jobSpec.AutoSavepointSeconds)) - return time.Now().After(nextTime), v1beta1.SavepointTriggerReasonScheduled -} -// Convert raw time to object and add `addedSeconds` to it, -// getting a time object for the parsed `rawTime` with `addedSeconds` added to it. -func getTimeAfterAddedSeconds(rawTime string, addedSeconds int64) time.Time { - var tc = &TimeConverter{} - var lastTriggerTime = time.Time{} - if len(rawTime) != 0 { - lastTriggerTime = tc.FromString(rawTime) - } - return lastTriggerTime.Add(time.Duration(addedSeconds * int64(time.Second))) + return "" } // Trigger savepoint for a job then return savepoint status to update. @@ -820,7 +804,7 @@ func (reconciler *ClusterReconciler) takeSavepointAsync(jobID string, triggerRea var message string var err error - log.Info("Trigger savepoint.", "jobID", jobID) + log.Info(fmt.Sprintf("Trigger savepoint for %s", triggerReason), "jobID", jobID) triggerID, err = reconciler.flinkClient.TakeSavepointAsync(apiBaseURL, jobID, *cluster.Spec.Job.SavepointsDir) if err != nil { // limit message size to 1KiB @@ -833,13 +817,9 @@ func (reconciler *ClusterReconciler) takeSavepointAsync(jobID string, triggerRea triggerSuccess = true log.Info("Savepoint is triggered successfully.", "jobID", jobID, "triggerID", triggerID) } - newSavepointStatus := getTriggeredSavepointStatus(jobID, triggerID, triggerReason, message, triggerSuccess) - requestedSavepoint := reconciler.observed.cluster.Status.Savepoint - // When savepoint was requested, maintain the requested time - if requestedSavepoint != nil && requestedSavepoint.State == v1beta1.SavepointStateNotTriggered { - newSavepointStatus.RequestTime = requestedSavepoint.RequestTime - } - return &newSavepointStatus, err + newSavepointStatus := getNewSavepointStatus(jobID, triggerID, triggerReason, message, triggerSuccess) + + return newSavepointStatus, err } // Takes savepoint for a job then update job status with the info. @@ -863,74 +843,9 @@ func (reconciler *ClusterReconciler) takeSavepoint( log.Info("Failed to take savepoint.", "jobID", jobID) } - statusUpdateErr := reconciler.updateSavepointStatus(status) - if statusUpdateErr != nil { - log.Error( - statusUpdateErr, "Failed to update savepoint status.", "error", statusUpdateErr) - } - return err } -func (reconciler *ClusterReconciler) updateSavepointStatus( - savepointStatus flinkclient.SavepointStatus) error { - var cluster = v1beta1.FlinkCluster{} - reconciler.observed.cluster.DeepCopyInto(&cluster) - if savepointStatus.IsSuccessful() { - var jobStatus = cluster.Status.Components.Job - jobStatus.SavepointGeneration++ - jobStatus.LastSavepointTriggerID = savepointStatus.TriggerID - jobStatus.SavepointLocation = savepointStatus.Location - - // TODO: LastSavepointTime should be set with the timestamp generated in job manager. - // Currently savepoint complete timestamp is not included in savepoints API response. - // Whereas checkpoint API returns the timestamp latest_ack_timestamp. - // Note: https://ci.apache.org/projects/flink/flink-docs-stable/ops/rest_api.html#jobs-jobid-checkpoints-details-checkpointid - setTimestamp(&jobStatus.LastSavepointTime) - setTimestamp(&cluster.Status.LastUpdateTime) - } - // case in which savepointing is triggered by control annotation - var controlStatus = cluster.Status.Control - if controlStatus != nil && controlStatus.Name == v1beta1.ControlNameSavepoint && - controlStatus.State == v1beta1.ControlStateProgressing { - if controlStatus.Details == nil { - controlStatus.Details = make(map[string]string) - } - var retries, err = getRetryCount(controlStatus.Details) - if err == nil { - if savepointStatus.IsFailed() || retries != "1" { - controlStatus.Details[ControlRetries] = retries - } - } else { - reconciler.log.Error(err, "failed to get retries from control status", "control status", controlStatus) - } - controlStatus.Details[ControlSavepointTriggerID] = savepointStatus.TriggerID - controlStatus.Details[ControlJobID] = savepointStatus.JobID - setTimestamp(&controlStatus.UpdateTime) - } - return reconciler.k8sClient.Status().Update(reconciler.context, &cluster) -} - -// If job cancellation is failed, fill the status message with error message. -// Then, the state will be transited to the failed by the updater. -func getFailedCancelStatus(cancelErr error) *v1beta1.FlinkClusterControlStatus { - var state string - var message string - var now string - setTimestamp(&now) - state = v1beta1.ControlStateProgressing - // limit message size to 1KiB - if message = cancelErr.Error(); len(message) > 1024 { - message = message[:1024] + "..." - } - return &v1beta1.FlinkClusterControlStatus{ - Name: v1beta1.ControlNameJobCancel, - State: state, - UpdateTime: now, - Message: message, - } -} - func (reconciler *ClusterReconciler) updateStatus(ss **v1beta1.SavepointStatus, cs **v1beta1.FlinkClusterControlStatus) { var log = reconciler.log @@ -1028,3 +943,34 @@ func (reconciler *ClusterReconciler) changeJobStateToUpdating() error { err := reconciler.k8sClient.Status().Update(reconciler.context, clusterClone) return err } + +// If job cancellation is failed, fill the status message with error message. +// Then, the state will be transited to the failed by the updater. +func getFailedCancelStatus(cancelErr error) *v1beta1.FlinkClusterControlStatus { + var state string + var message string + var now string + setTimestamp(&now) + state = v1beta1.ControlStateInProgress + // limit message size to 1KiB + if message = cancelErr.Error(); len(message) > 1024 { + message = message[:1024] + "..." + } + return &v1beta1.FlinkClusterControlStatus{ + Name: v1beta1.ControlNameJobCancel, + State: state, + UpdateTime: now, + Message: message, + } +} + +// Convert raw time to object and add `addedSeconds` to it, +// getting a time object for the parsed `rawTime` with `addedSeconds` added to it. +func getTimeAfterAddedSeconds(rawTime string, addedSeconds int64) time.Time { + var tc = &TimeConverter{} + var lastTriggerTime = time.Time{} + if len(rawTime) != 0 { + lastTriggerTime = tc.FromString(rawTime) + } + return lastTriggerTime.Add(time.Duration(addedSeconds * int64(time.Second))) +} diff --git a/controllers/flinkcluster_updater.go b/controllers/flinkcluster_updater.go index bed50a32..43cdf172 100644 --- a/controllers/flinkcluster_updater.go +++ b/controllers/flinkcluster_updater.go @@ -458,134 +458,17 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( } // Update savepoint status if it is in progress or requested. - if recorded.Savepoint != nil { - status.Savepoint = updater.getSavepointStatus( - observed.savepoint, - recorded.Savepoint, - recorded.Components.Job, - updater.getFlinkJobID()) - } + status.Savepoint = updater.getUpdatedSavepointStatus( + observed.savepoint, + recorded.Savepoint, + recorded.Components.Job, + updater.getFlinkJobID()) // User requested control - var userControl = observed.cluster.Annotations[v1beta1.ControlAnnotation] + status.Control = getControlStatus(observed.cluster, &status, recorded) - // Update job control status in progress - var controlStatus *v1beta1.FlinkClusterControlStatus - if recorded.Control != nil && userControl == recorded.Control.Name && - recorded.Control.State == v1beta1.ControlStateProgressing { - controlStatus = recorded.Control.DeepCopy() - var newSavepointStatus = status.Savepoint - switch recorded.Control.Name { - case v1beta1.ControlNameJobCancel: - if status.Components.Job.State == v1beta1.JobStateCancelled { - controlStatus.State = v1beta1.ControlStateSucceeded - setTimestamp(&controlStatus.UpdateTime) - } else if isJobTerminated(observed.cluster.Spec.Job.RestartPolicy, recorded.Components.Job) { - controlStatus.Message = "Aborted job cancellation: Job is terminated." - controlStatus.State = v1beta1.ControlStateFailed - setTimestamp(&controlStatus.UpdateTime) - } else if newSavepointStatus != nil && newSavepointStatus.State == v1beta1.SavepointStateFailed { - controlStatus.Message = "Aborted job cancellation: failed to create savepoint." - controlStatus.State = v1beta1.ControlStateFailed - setTimestamp(&controlStatus.UpdateTime) - } else if recorded.Control.Message != "" { - controlStatus.State = v1beta1.ControlStateFailed - setTimestamp(&controlStatus.UpdateTime) - } - case v1beta1.ControlNameSavepoint: - if newSavepointStatus != nil { - if newSavepointStatus.State == v1beta1.SavepointStateSucceeded { - controlStatus.State = v1beta1.ControlStateSucceeded - setTimestamp(&controlStatus.UpdateTime) - } else if newSavepointStatus.State == v1beta1.SavepointStateFailed || newSavepointStatus.State == v1beta1.SavepointStateTriggerFailed { - controlStatus.State = v1beta1.ControlStateFailed - setTimestamp(&controlStatus.UpdateTime) - } - } - } - // aborted by max retry reach - var retries = controlStatus.Details[ControlRetries] - if retries == ControlMaxRetries { - controlStatus.Message = fmt.Sprintf("Aborted control %v. The maximum number of retries has been reached.", controlStatus.Name) - controlStatus.State = v1beta1.ControlStateFailed - setTimestamp(&controlStatus.UpdateTime) - } - } else if userControl != "" { - // Handle new user control. - updater.log.Info("New user control requested: " + userControl) - if userControl != v1beta1.ControlNameJobCancel && userControl != v1beta1.ControlNameSavepoint { - if userControl != "" { - updater.log.Info(fmt.Sprintf(v1beta1.InvalidControlAnnMsg, v1beta1.ControlAnnotation, userControl)) - } - } else if recorded.Control != nil && recorded.Control.State == v1beta1.ControlStateProgressing { - updater.log.Info(fmt.Sprintf(v1beta1.ControlChangeWarnMsg, v1beta1.ControlAnnotation), "current control", recorded.Control.Name, "new control", userControl) - } else { - switch userControl { - case v1beta1.ControlNameSavepoint: - if observed.cluster.Spec.Job.SavepointsDir == nil || *observed.cluster.Spec.Job.SavepointsDir == "" { - updater.log.Info(fmt.Sprintf(v1beta1.InvalidSavepointDirMsg, v1beta1.ControlAnnotation)) - break - } else if isJobStopped(observed.cluster.Status.Components.Job) { - updater.log.Info(fmt.Sprintf(v1beta1.InvalidJobStateForSavepointMsg, v1beta1.ControlAnnotation)) - break - } - // Although a savepoint is in progress, if user requested savepoint, new savepoint should be triggered. - status.Savepoint = getRequestedSavepointStatus(v1beta1.SavepointTriggerReasonUserRequested) - controlStatus = getNewUserControlStatus(userControl) - updater.log.Info("Marked to take new savepoint user requested.") - case v1beta1.ControlNameJobCancel: - if isJobTerminated(observed.cluster.Spec.Job.RestartPolicy, recorded.Components.Job) { - updater.log.Info(fmt.Sprintf(v1beta1.InvalidJobStateForJobCancelMsg, v1beta1.ControlAnnotation)) - break - } - // Savepoint for job-cancel - var observedSavepoint = observed.cluster.Status.Savepoint - if observedSavepoint == nil || - (observedSavepoint.State != v1beta1.SavepointStateInProgress && observedSavepoint.State != v1beta1.SavepointStateNotTriggered) { - status.Savepoint = getRequestedSavepointStatus(v1beta1.SavepointTriggerReasonJobCancel) - updater.log.Info("Marked to take new savepoint for job cancel.") - } else { - updater.log.Info("There is a savepoint in progress. No need to trigger new savepoint for job cancel.") - } - controlStatus = getNewUserControlStatus(userControl) - } - } - } - // Maintain control status if there is no change. - if recorded.Control != nil && controlStatus == nil { - controlStatus = recorded.Control.DeepCopy() - } - status.Control = controlStatus - - // Handle update. - switch updateState { - case UpdateStatePreparing: - // Even if savepoint has been created for update already, we check the age of savepoint continually. - // If created savepoint is old and savepoint can be triggered, we should take savepoint again. - // (e.g., for the case update is not progressed by accidents like network partition) - var jobSpec = observed.cluster.Spec.Job - var jobStatus = status.Components.Job - if !isSavepointUpToDate(observed.observeTime, jobSpec, jobStatus) && - canTakeSavepoint(*observed.cluster) && - (recorded.Savepoint == nil || recorded.Savepoint.State != v1beta1.SavepointStateNotTriggered) { - // TODO: apply exponential backoff retry - // If failed to take savepoint, retry after SavepointRequestRetryIntervalSec. - if recorded.Savepoint != nil && - !hasTimeElapsed(recorded.Savepoint.RequestTime, time.Now(), SavepointRequestRetryIntervalSec) { - updater.log.Info(fmt.Sprintf("Will retry to trigger savepoint for update, in %v seconds because previous request was failed", SavepointRequestRetryIntervalSec)) - } else { - // Mark to take new savepoint for update. - status.Savepoint = getRequestedSavepointStatus(v1beta1.SavepointTriggerReasonUpdate) - updater.log.Info("Marked to take new savepoint for update.") - } - } else if recorded.Savepoint != nil && recorded.Savepoint.State == v1beta1.SavepointStateInProgress { - updater.log.Info("Savepoint for update is in progress.") - } else { - updater.log.Info("Job is stopping for update.") - } - case UpdateStateInProgress: - updater.log.Info("Update is in progress.") - case UpdateStateFinished: + // Update finished + if updateState == UpdateStateFinished { status.CurrentRevision = observed.cluster.Status.NextRevision updater.log.Info("Finished update.") } @@ -893,14 +776,18 @@ func (updater *ClusterStatusUpdater) clearControlAnnotation(newControlStatus *v1 return nil } -func (updater *ClusterStatusUpdater) getSavepointStatus( +func (updater *ClusterStatusUpdater) getUpdatedSavepointStatus( observedSavepoint *Savepoint, recordedSavepointStatus *v1beta1.SavepointStatus, recordedJobStatus *v1beta1.JobStatus, flinkJobID *string) *v1beta1.SavepointStatus { + if recordedSavepointStatus == nil { + return nil + } + var savepointStatus = recordedSavepointStatus.DeepCopy() var errMsg string - if recordedSavepointStatus.State == v1beta1.SavepointStateInProgress && observedSavepoint != nil { + if savepointStatus.State == v1beta1.SavepointStateInProgress && observedSavepoint != nil { switch { case observedSavepoint.IsSuccessful(): savepointStatus.State = v1beta1.SavepointStateSucceeded @@ -914,7 +801,7 @@ func (updater *ClusterStatusUpdater) getSavepointStatus( } } } - if savepointStatus.State == v1beta1.SavepointStateNotTriggered || savepointStatus.State == v1beta1.SavepointStateInProgress { + if savepointStatus.State == v1beta1.SavepointStateInProgress { switch { case isJobStopped(recordedJobStatus): errMsg = "Flink job is stopped." @@ -941,6 +828,70 @@ func (updater *ClusterStatusUpdater) getSavepointStatus( return savepointStatus } +func getControlStatus(cluster *v1beta1.FlinkCluster, + new *v1beta1.FlinkClusterStatus, + recorded *v1beta1.FlinkClusterStatus) *v1beta1.FlinkClusterControlStatus { + var userControl = cluster.Annotations[v1beta1.ControlAnnotation] + var controlStatus *v1beta1.FlinkClusterControlStatus + var controlRequest = getNewUserControlRequest(cluster) + + // New control status + if controlRequest != "" { + controlStatus = getUserControlStatus(controlRequest, v1beta1.ControlStateRequested) + return controlStatus + } + + // Update control status in progress + if recorded.Control != nil && userControl == recorded.Control.Name && + recorded.Control.State == v1beta1.ControlStateInProgress { + controlStatus = recorded.Control.DeepCopy() + switch recorded.Control.Name { + case v1beta1.ControlNameJobCancel: + if new.Components.Job.State == v1beta1.JobStateCancelled { + controlStatus.State = v1beta1.ControlStateSucceeded + setTimestamp(&controlStatus.UpdateTime) + } else if isJobTerminated(cluster.Spec.Job.RestartPolicy, recorded.Components.Job) { + controlStatus.Message = "Aborted job cancellation: job is terminated already." + controlStatus.State = v1beta1.ControlStateFailed + setTimestamp(&controlStatus.UpdateTime) + } else if new.Savepoint != nil && new.Savepoint.State == v1beta1.SavepointStateFailed { + controlStatus.Message = "Aborted job cancellation: failed to take savepoint." + controlStatus.State = v1beta1.ControlStateFailed + setTimestamp(&controlStatus.UpdateTime) + } else if recorded.Control.Message != "" { + controlStatus.State = v1beta1.ControlStateFailed + setTimestamp(&controlStatus.UpdateTime) + } + case v1beta1.ControlNameSavepoint: + if new.Savepoint != nil { + if new.Savepoint.State == v1beta1.SavepointStateSucceeded { + controlStatus.State = v1beta1.ControlStateSucceeded + setTimestamp(&controlStatus.UpdateTime) + } else if new.Savepoint.State == v1beta1.SavepointStateFailed || new.Savepoint.State == v1beta1.SavepointStateTriggerFailed { + controlStatus.State = v1beta1.ControlStateFailed + setTimestamp(&controlStatus.UpdateTime) + } + } + } + // Aborted by max retry reach + var retries = controlStatus.Details[ControlRetries] + if retries == ControlMaxRetries { + controlStatus.Message = fmt.Sprintf("Aborted control %v. The maximum number of retries has been reached.", controlStatus.Name) + controlStatus.State = v1beta1.ControlStateFailed + setTimestamp(&controlStatus.UpdateTime) + } + return controlStatus + } + + // Maintain control status if there is no change. + if recorded.Control != nil && controlStatus == nil { + controlStatus = recorded.Control.DeepCopy() + return controlStatus + } + + return nil +} + func getStatefulSetState(statefulSet *appsv1.StatefulSet) string { if statefulSet.Status.ReadyReplicas >= *statefulSet.Spec.Replicas { return v1beta1.ComponentStateReady diff --git a/controllers/flinkcluster_util.go b/controllers/flinkcluster_util.go index b1ebcff4..5973d5a0 100644 --- a/controllers/flinkcluster_util.go +++ b/controllers/flinkcluster_util.go @@ -37,10 +37,8 @@ import ( ) const ( - ControlSavepointTriggerID = "SavepointTriggerID" - ControlJobID = "jobID" - ControlRetries = "retries" - ControlMaxRetries = "3" + ControlRetries = "retries" + ControlMaxRetries = "3" RevisionNameLabel = "flinkoperator.k8s.io/revision-name" @@ -140,7 +138,7 @@ func canTakeSavepoint(cluster v1beta1.FlinkCluster) bool { var savepointStatus = cluster.Status.Savepoint var jobStatus = cluster.Status.Components.Job return jobSpec != nil && jobSpec.SavepointsDir != nil && - jobStatus.State == v1beta1.JobStateRunning && + !isJobStopped(jobStatus) && (savepointStatus == nil || savepointStatus.State != v1beta1.SavepointStateInProgress) } @@ -272,40 +270,42 @@ func getRetryCount(data map[string]string) (string, error) { return retries, err } -func getNewUserControlStatus(controlName string) *v1beta1.FlinkClusterControlStatus { +func getNewUserControlRequest(cluster *v1beta1.FlinkCluster) string { + var userControl = cluster.Annotations[v1beta1.ControlAnnotation] + var recorded = cluster.Status + if recorded.Control == nil || recorded.Control.State != v1beta1.ControlStateInProgress { + return userControl + } + return "" +} + +func getUserControlStatus(controlName string, state string) *v1beta1.FlinkClusterControlStatus { var controlStatus = new(v1beta1.FlinkClusterControlStatus) controlStatus.Name = controlName - controlStatus.State = v1beta1.ControlStateProgressing + controlStatus.State = state setTimestamp(&controlStatus.UpdateTime) return controlStatus } -func getTriggeredSavepointStatus(jobID string, triggerID string, triggerReason string, message string, triggerSuccess bool) v1beta1.SavepointStatus { - var savepointStatus = v1beta1.SavepointStatus{} +func getNewSavepointStatus(jobID string, triggerID string, triggerReason string, message string, triggerSuccess bool) *v1beta1.SavepointStatus { + var savepointState string var now string setTimestamp(&now) - savepointStatus.JobID = jobID - savepointStatus.TriggerID = triggerID - savepointStatus.TriggerReason = triggerReason - savepointStatus.TriggerTime = now - savepointStatus.RequestTime = now - savepointStatus.Message = message if triggerSuccess { - savepointStatus.State = v1beta1.SavepointStateInProgress + savepointState = v1beta1.SavepointStateInProgress } else { - savepointStatus.State = v1beta1.SavepointStateTriggerFailed + savepointState = v1beta1.SavepointStateTriggerFailed } - return savepointStatus -} - -func getRequestedSavepointStatus(triggerReason string) *v1beta1.SavepointStatus { - var now string - setTimestamp(&now) - return &v1beta1.SavepointStatus{ - State: v1beta1.SavepointStateNotTriggered, + var savepointStatus = &v1beta1.SavepointStatus{ + JobID: jobID, + TriggerID: triggerID, TriggerReason: triggerReason, - RequestTime: now, + TriggerTime: now, + UpdateTime: now, + Message: message, + State: savepointState, } + return savepointStatus } func getControlEvent(status v1beta1.FlinkClusterControlStatus) (eventType string, eventReason string, eventMessage string) { @@ -314,7 +314,7 @@ func getControlEvent(status v1beta1.FlinkClusterControlStatus) (eventType string msg = msg[:100] + "..." } switch status.State { - case v1beta1.ControlStateProgressing: + case v1beta1.ControlStateInProgress: eventType = corev1.EventTypeNormal eventReason = "ControlRequested" eventMessage = fmt.Sprintf("Requested new user control %v", status.Name) From 8564ece9b18958e139517b5519ed6a68f1d8b66b Mon Sep 17 00:00:00 2001 From: elanv Date: Thu, 18 Mar 2021 01:40:23 +0900 Subject: [PATCH 3/6] Applied new job stop, update, restart routine --- api/v1beta1/flinkcluster_default.go | 6 +- api/v1beta1/flinkcluster_default_test.go | 38 +- api/v1beta1/flinkcluster_types.go | 106 +++-- api/v1beta1/flinkcluster_validate.go | 22 +- api/v1beta1/zz_generated.deepcopy.go | 40 +- .../flinkoperator.k8s.io_flinkclusters.yaml | 292 +++--------- controllers/flinkclient/flink_client.go | 65 ++- controllers/flinkcluster_converter.go | 40 +- controllers/flinkcluster_converter_test.go | 6 +- controllers/flinkcluster_observer.go | 330 +++++++------ controllers/flinkcluster_reconciler.go | 396 +++++++--------- controllers/flinkcluster_updater.go | 442 ++++++++++-------- controllers/flinkcluster_util.go | 245 +++++----- controllers/flinkcluster_util_test.go | 384 ++++----------- 14 files changed, 1069 insertions(+), 1343 deletions(-) diff --git a/api/v1beta1/flinkcluster_default.go b/api/v1beta1/flinkcluster_default.go index fbe3d17a..744cd0ae 100644 --- a/api/v1beta1/flinkcluster_default.go +++ b/api/v1beta1/flinkcluster_default.go @@ -128,9 +128,9 @@ func _SetJobDefault(jobSpec *JobSpec) { AfterJobCancelled: CleanupActionDeleteCluster, } } - if jobSpec.SavepointMaxAgeForUpdateSeconds == nil { - jobSpec.SavepointMaxAgeForUpdateSeconds = new(int32) - *jobSpec.SavepointMaxAgeForUpdateSeconds = 300 + if jobSpec.MaxStateAgeToRestoreSeconds == nil { + jobSpec.MaxStateAgeToRestoreSeconds = new(int32) + *jobSpec.MaxStateAgeToRestoreSeconds = 300 } } diff --git a/api/v1beta1/flinkcluster_default_test.go b/api/v1beta1/flinkcluster_default_test.go index 397d46fc..c4608d13 100644 --- a/api/v1beta1/flinkcluster_default_test.go +++ b/api/v1beta1/flinkcluster_default_test.go @@ -53,7 +53,7 @@ func TestSetDefault(t *testing.T) { var defaultJobParallelism = int32(1) var defaultJobNoLoggingToStdout = false var defaultJobRestartPolicy = JobRestartPolicyNever - var defaultJobSavepointMaxAgeForUpdateSeconds = int32(300) + var defaultMaxStateAgeToRestoreSeconds = int32(300) var defaultMemoryOffHeapRatio = int32(25) var defaultMemoryOffHeapMin = resource.MustParse("600M") var defaultRecreateOnUpdate = true @@ -99,11 +99,11 @@ func TestSetDefault(t *testing.T) { SecurityContext: nil, }, Job: &JobSpec{ - AllowNonRestoredState: &defaultJobAllowNonRestoredState, - Parallelism: &defaultJobParallelism, - NoLoggingToStdout: &defaultJobNoLoggingToStdout, - RestartPolicy: &defaultJobRestartPolicy, - SavepointMaxAgeForUpdateSeconds: &defaultJobSavepointMaxAgeForUpdateSeconds, + AllowNonRestoredState: &defaultJobAllowNonRestoredState, + Parallelism: &defaultJobParallelism, + NoLoggingToStdout: &defaultJobNoLoggingToStdout, + RestartPolicy: &defaultJobRestartPolicy, + MaxStateAgeToRestoreSeconds: &defaultMaxStateAgeToRestoreSeconds, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: "DeleteCluster", AfterJobFails: "KeepCluster", @@ -143,7 +143,7 @@ func TestSetNonDefault(t *testing.T) { var jobParallelism = int32(2) var jobNoLoggingToStdout = true var jobRestartPolicy = JobRestartPolicyFromSavepointOnFailure - var jobSavepointMaxAgeForUpdateSeconds = int32(1000) + var jobMaxStateAgeToRestoreSeconds = int32(1000) var memoryOffHeapRatio = int32(50) var memoryOffHeapMin = resource.MustParse("600M") var recreateOnUpdate = false @@ -194,12 +194,12 @@ func TestSetNonDefault(t *testing.T) { SecurityContext: &securityContext, }, Job: &JobSpec{ - AllowNonRestoredState: &jobAllowNonRestoredState, - Parallelism: &jobParallelism, - NoLoggingToStdout: &jobNoLoggingToStdout, - RestartPolicy: &jobRestartPolicy, - SavepointMaxAgeForUpdateSeconds: &jobSavepointMaxAgeForUpdateSeconds, - SecurityContext: &securityContext, + AllowNonRestoredState: &jobAllowNonRestoredState, + Parallelism: &jobParallelism, + NoLoggingToStdout: &jobNoLoggingToStdout, + RestartPolicy: &jobRestartPolicy, + MaxStateAgeToRestoreSeconds: &jobMaxStateAgeToRestoreSeconds, + SecurityContext: &securityContext, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: "DeleteTaskManagers", AfterJobFails: "DeleteCluster", @@ -260,12 +260,12 @@ func TestSetNonDefault(t *testing.T) { SecurityContext: &securityContext, }, Job: &JobSpec{ - AllowNonRestoredState: &jobAllowNonRestoredState, - Parallelism: &jobParallelism, - NoLoggingToStdout: &jobNoLoggingToStdout, - RestartPolicy: &jobRestartPolicy, - SavepointMaxAgeForUpdateSeconds: &jobSavepointMaxAgeForUpdateSeconds, - SecurityContext: &securityContext, + AllowNonRestoredState: &jobAllowNonRestoredState, + Parallelism: &jobParallelism, + NoLoggingToStdout: &jobNoLoggingToStdout, + RestartPolicy: &jobRestartPolicy, + MaxStateAgeToRestoreSeconds: &jobMaxStateAgeToRestoreSeconds, + SecurityContext: &securityContext, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: "DeleteTaskManagers", AfterJobFails: "DeleteCluster", diff --git a/api/v1beta1/flinkcluster_types.go b/api/v1beta1/flinkcluster_types.go index 230d7793..2789c37e 100644 --- a/api/v1beta1/flinkcluster_types.go +++ b/api/v1beta1/flinkcluster_types.go @@ -43,15 +43,17 @@ const ( // JobState defines states for a Flink job deployment. const ( - JobStatePending = "Pending" - JobStateRunning = "Running" - JobStateUpdating = "Updating" - JobStateSucceeded = "Succeeded" - JobStateFailed = "Failed" - JobStateCancelled = "Cancelled" - JobStateSuspended = "Suspended" - JobStateUnknown = "Unknown" - JobStateLost = "Lost" + JobStatePending = "Pending" + JobStateUpdating = "Updating" + JobStateRestarting = "Restarting" + JobStateDeploying = "Deploying" + JobStateDeployFailed = "DeployFailed" + JobStateRunning = "Running" + JobStateSucceeded = "Succeeded" + JobStateCancelled = "Cancelled" + JobStateFailed = "Failed" + JobStateLost = "Lost" + JobStateUnknown = "Unknown" ) // AccessScope defines the access scope of JobManager service. @@ -99,8 +101,8 @@ const ( SavepointStateSucceeded = "Succeeded" SavepointTriggerReasonUserRequested = "user requested" - SavepointTriggerReasonScheduled = "scheduled" SavepointTriggerReasonJobCancel = "job cancel" + SavepointTriggerReasonScheduled = "scheduled" SavepointTriggerReasonUpdate = "update" ) @@ -347,14 +349,14 @@ type JobSpec struct { // Allow non-restored state, default: false. AllowNonRestoredState *bool `json:"allowNonRestoredState,omitempty"` - // Should take savepoint before updating the job, default: true. - TakeSavepointOnUpdate *bool `json:"takeSavepointOnUpdate,omitempty"` - // Savepoints dir where to store savepoints of the job. SavepointsDir *string `json:"savepointsDir,omitempty"` - // Max age of savepoint allowed to progress update. - SavepointMaxAgeForUpdateSeconds *int32 `json:"savepointMaxAgeForUpdateSeconds,omitempty"` + // Should take savepoint before updating job, default: true. + TakeSavepointOnUpdate *bool `json:"takeSavepointOnUpdate,omitempty"` + + // Maximum age of the savepoint that a job can be restored when the job is restarted or updated from stopped state, default: 300 + MaxStateAgeToRestoreSeconds *int32 `json:"maxStateAgeToRestoreSeconds,omitempty"` // Automatically take a savepoint to the `savepointsDir` every n seconds. AutoSavepointSeconds *int32 `json:"autoSavepointSeconds,omitempty"` @@ -557,7 +559,7 @@ type JobStatus struct { // The ID of the Flink job. ID string `json:"id,omitempty"` - // The state of the Kubernetes job. + // The state of the Flink job deployment. State string `json:"state"` // The actual savepoint from which this job started. @@ -573,20 +575,26 @@ type JobStatus struct { // Savepoint location. SavepointLocation string `json:"savepointLocation,omitempty"` - // Last savepoint trigger ID. - LastSavepointTriggerID string `json:"lastSavepointTriggerID,omitempty"` + // Last successful savepoint completed timestamp. + SavepointTime string `json:"savepointTime,omitempty"` - // Last successful or failed savepoint operation timestamp. - LastSavepointTime string `json:"lastSavepointTime,omitempty"` + // The savepoint is the final state of the job. + FinalSavepoint bool `json:"finalSavepoint,omitempty"` + + // The timestamp of the Flink job deployment that creating job submitter. + DeployTime string `json:"deployTime,omitempty"` // The Flink job started timestamp. StartTime string `json:"startTime,omitempty"` + // The Flink job ended timestamp. + EndTime string `json:"endTime,omitempty"` + // The number of restarts. RestartCount int32 `json:"restartCount,omitempty"` } -// SavepointStatus defines the status of savepoint progress +// SavepointStatus is the status of savepoint progress. type SavepointStatus struct { // The ID of the Flink job. JobID string `json:"jobID,omitempty"` @@ -600,7 +608,7 @@ type SavepointStatus struct { // Savepoint triggered reason. TriggerReason string `json:"triggerReason,omitempty"` - // Savepoint requested time. + // Savepoint status update time. UpdateTime string `json:"requestTime,omitempty"` // Savepoint state. @@ -610,6 +618,27 @@ type SavepointStatus struct { Message string `json:"message,omitempty"` } +type RevisionStatus struct { + // When the controller creates new ControllerRevision, it generates hash string from the FlinkCluster spec + // which is to be stored in ControllerRevision and uses it to compose the ControllerRevision name. + // Then the controller updates nextRevision to the ControllerRevision name. + // When update process is completed, the controller updates currentRevision as nextRevision. + // currentRevision and nextRevision is composed like this: + // -- + // e.g., myflinkcluster-c464ff7-5 + + // CurrentRevision indicates the version of FlinkCluster. + CurrentRevision string `json:"currentRevision,omitempty"` + + // NextRevision indicates the version of FlinkCluster updating. + NextRevision string `json:"nextRevision,omitempty"` + + // collisionCount is the count of hash collisions for the FlinkCluster. The controller + // uses this field as a collision avoidance mechanism when it needs to create the name for the + // newest ControllerRevision. + CollisionCount *int32 `json:"collisionCount,omitempty"` +} + // JobManagerIngressStatus defines the status of a JobManager ingress. type JobManagerIngressStatus struct { // The name of the Kubernetes ingress resource. @@ -645,30 +674,14 @@ type FlinkClusterStatus struct { // The status of the components. Components FlinkClusterComponentsStatus `json:"components"` - // The status of control requested by user + // The status of control requested by user. Control *FlinkClusterControlStatus `json:"control,omitempty"` - // The status of savepoint progress + // The status of savepoint progress. Savepoint *SavepointStatus `json:"savepoint,omitempty"` - // When the controller creates new ControllerRevision, it generates hash string from the FlinkCluster spec - // which is to be stored in ControllerRevision and uses it to compose the ControllerRevision name. - // Then the controller updates nextRevision to the ControllerRevision name. - // When update process is completed, the controller updates currentRevision as nextRevision. - // currentRevision and nextRevision is composed like this: - // -- - // e.g., myflinkcluster-c464ff7-5 - - // CurrentRevision indicates the version of FlinkCluster. - CurrentRevision string `json:"currentRevision,omitempty"` - - // NextRevision indicates the version of FlinkCluster updating. - NextRevision string `json:"nextRevision,omitempty"` - - // collisionCount is the count of hash collisions for the FlinkCluster. The controller - // uses this field as a collision avoidance mechanism when it needs to create the name for the - // newest ControllerRevision. - CollisionCount *int32 `json:"collisionCount,omitempty"` + // The status of revision. + Revision RevisionStatus `json:"revision"` // Last update timestamp for this status. LastUpdateTime string `json:"lastUpdateTime,omitempty"` @@ -698,3 +711,12 @@ type FlinkClusterList struct { func init() { SchemeBuilder.Register(&FlinkCluster{}, &FlinkClusterList{}) } + +func (j *JobStatus) isJobStopped() bool { + return j != nil && + (j.State == JobStateSucceeded || + j.State == JobStateCancelled || + j.State == JobStateFailed || + j.State == JobStateLost || + j.State == JobStateDeployFailed) +} diff --git a/api/v1beta1/flinkcluster_validate.go b/api/v1beta1/flinkcluster_validate.go index 949154a3..2a35388f 100644 --- a/api/v1beta1/flinkcluster_validate.go +++ b/api/v1beta1/flinkcluster_validate.go @@ -132,7 +132,7 @@ func (v *Validator) checkControlAnnotations(old *FlinkCluster, new *FlinkCluster return fmt.Errorf(SessionClusterWarnMsg, ControlNameSavepoint, ControlAnnotation) } else if old.Spec.Job.SavepointsDir == nil || *old.Spec.Job.SavepointsDir == "" { return fmt.Errorf(InvalidSavepointDirMsg, ControlAnnotation) - } else if jobStatus == nil || isJobStopped(old.Status.Components.Job) { + } else if jobStatus == nil || jobStatus.isJobStopped() { return fmt.Errorf(InvalidJobStateForSavepointMsg, ControlAnnotation) } default: @@ -207,6 +207,7 @@ func (v *Validator) checkSavepointGeneration( // Validate job update. func (v *Validator) validateJobUpdate(old *FlinkCluster, new *FlinkCluster) error { + var jobStatus = old.Status.Components.Job switch { case old.Spec.Job == nil && new.Spec.Job == nil: return nil @@ -219,6 +220,16 @@ func (v *Validator) validateJobUpdate(old *FlinkCluster, new *FlinkCluster) erro case old.Spec.Job.SavepointsDir != nil && *old.Spec.Job.SavepointsDir != "" && (new.Spec.Job.SavepointsDir == nil || *new.Spec.Job.SavepointsDir == ""): return fmt.Errorf("removing savepointsDir is not allowed") + case jobStatus != nil && jobStatus.isJobStopped(): + if jobStatus.FinalSavepoint { + return nil + } + var shouldTakeSavepoint = (new.Spec.Job.TakeSavepointOnUpdate == nil || *new.Spec.Job.TakeSavepointOnUpdate) && + (new.Spec.Job.FromSavepoint == nil || *new.Spec.Job.FromSavepoint == "") + if shouldTakeSavepoint { + return fmt.Errorf("cannot update because job is stoppped without final savepoint," + + "to proceed update, you need to set takeSavepointOnUpdate false or provide fromSavepoint") + } } return nil } @@ -520,13 +531,6 @@ func shouldRestartJob( len(jobStatus.SavepointLocation) > 0 } -func isJobStopped(status *JobStatus) bool { - return status != nil && - (status.State == JobStateSucceeded || - status.State == JobStateFailed || - status.State == JobStateCancelled) -} - func isJobTerminated(restartPolicy *JobRestartPolicy, jobStatus *JobStatus) bool { - return isJobStopped(jobStatus) && !shouldRestartJob(restartPolicy, jobStatus) + return jobStatus.isJobStopped() && !shouldRestartJob(restartPolicy, jobStatus) } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 62d3229f..93e8f5ae 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -260,11 +260,7 @@ func (in *FlinkClusterStatus) DeepCopyInto(out *FlinkClusterStatus) { *out = new(SavepointStatus) **out = **in } - if in.CollisionCount != nil { - in, out := &in.CollisionCount, &out.CollisionCount - *out = new(int32) - **out = **in - } + in.Revision.DeepCopyInto(&out.Revision) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FlinkClusterStatus. @@ -583,18 +579,18 @@ func (in *JobSpec) DeepCopyInto(out *JobSpec) { *out = new(bool) **out = **in } - if in.TakeSavepointOnUpdate != nil { - in, out := &in.TakeSavepointOnUpdate, &out.TakeSavepointOnUpdate - *out = new(bool) - **out = **in - } if in.SavepointsDir != nil { in, out := &in.SavepointsDir, &out.SavepointsDir *out = new(string) **out = **in } - if in.SavepointMaxAgeForUpdateSeconds != nil { - in, out := &in.SavepointMaxAgeForUpdateSeconds, &out.SavepointMaxAgeForUpdateSeconds + if in.TakeSavepointOnUpdate != nil { + in, out := &in.TakeSavepointOnUpdate, &out.TakeSavepointOnUpdate + *out = new(bool) + **out = **in + } + if in.MaxStateAgeToRestoreSeconds != nil { + in, out := &in.MaxStateAgeToRestoreSeconds, &out.MaxStateAgeToRestoreSeconds *out = new(int32) **out = **in } @@ -711,6 +707,26 @@ func (in *NamedPort) DeepCopy() *NamedPort { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RevisionStatus) DeepCopyInto(out *RevisionStatus) { + *out = *in + if in.CollisionCount != nil { + in, out := &in.CollisionCount, &out.CollisionCount + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RevisionStatus. +func (in *RevisionStatus) DeepCopy() *RevisionStatus { + if in == nil { + return nil + } + out := new(RevisionStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SavepointStatus) DeepCopyInto(out *SavepointStatus) { *out = *in diff --git a/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml b/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml index 8b6f61be..684709a7 100644 --- a/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml +++ b/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.2.4 creationTimestamp: null name: flinkclusters.flinkoperator.k8s.io spec: @@ -85,11 +85,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -221,11 +217,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -461,10 +453,6 @@ spec: - containerPort type: object type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -534,19 +522,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -714,6 +694,9 @@ spec: type: array jarFile: type: string + maxStateAgeToRestoreSeconds: + format: int32 + type: integer noLoggingToStdout: type: boolean parallelism: @@ -731,19 +714,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object restartPolicy: @@ -751,9 +726,6 @@ spec: savepointGeneration: format: int32 type: integer - savepointMaxAgeForUpdateSeconds: - format: int32 - type: integer savepointsDir: type: string securityContext: @@ -988,11 +960,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -1008,11 +976,7 @@ spec: medium: type: string sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object fc: properties: @@ -1237,11 +1201,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -1510,11 +1470,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -1750,10 +1706,6 @@ spec: - containerPort type: object type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -1823,19 +1775,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -2002,11 +1946,7 @@ spec: type: object type: array memoryOffHeapMin: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string memoryOffHeapRatio: format: int32 type: integer @@ -2044,19 +1984,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -2157,11 +2089,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -2397,10 +2325,6 @@ spec: - containerPort type: object type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -2470,19 +2394,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -2695,19 +2611,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object selector: @@ -2748,11 +2656,7 @@ spec: type: array capacity: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object conditions: items: @@ -2958,11 +2862,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -2978,11 +2878,7 @@ spec: medium: type: string sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object fc: properties: @@ -3207,11 +3103,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -3475,11 +3367,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -3715,10 +3603,6 @@ spec: - containerPort type: object type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -3788,19 +3672,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -3967,11 +3843,7 @@ spec: type: object type: array memoryOffHeapMin: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string memoryOffHeapRatio: format: int32 type: integer @@ -4006,19 +3878,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -4119,11 +3983,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -4359,10 +4219,6 @@ spec: - containerPort type: object type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -4432,19 +4288,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object securityContext: @@ -4657,19 +4505,11 @@ spec: properties: limits: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object requests: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object type: object selector: @@ -4710,11 +4550,7 @@ spec: type: array capacity: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object conditions: items: @@ -4920,11 +4756,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -4940,11 +4772,7 @@ spec: medium: type: string sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string type: object fc: properties: @@ -5169,11 +4997,7 @@ spec: containerName: type: string divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string resource: type: string required: @@ -5366,9 +5190,6 @@ spec: type: object status: properties: - collisionCount: - format: int32 - type: integer components: properties: configMap: @@ -5383,13 +5204,15 @@ spec: type: object job: properties: - fromSavepoint: + deployTime: type: string - id: + endTime: type: string - lastSavepointTime: + finalSavepoint: + type: boolean + fromSavepoint: type: string - lastSavepointTriggerID: + id: type: string name: type: string @@ -5401,6 +5224,8 @@ spec: type: integer savepointLocation: type: string + savepointTime: + type: string startTime: type: string state: @@ -5480,12 +5305,18 @@ spec: - state - updateTime type: object - currentRevision: - type: string lastUpdateTime: type: string - nextRevision: - type: string + revision: + properties: + collisionCount: + format: int32 + type: integer + currentRevision: + type: string + nextRevision: + type: string + type: object savepoint: properties: jobID: @@ -5509,6 +5340,7 @@ spec: type: string required: - components + - revision - state type: object required: diff --git a/controllers/flinkclient/flink_client.go b/controllers/flinkclient/flink_client.go index 81c0c8b4..5581d5ba 100644 --- a/controllers/flinkclient/flink_client.go +++ b/controllers/flinkclient/flink_client.go @@ -100,17 +100,40 @@ func (c *FlinkClient) StopJob( // TriggerSavepoint triggers an async savepoint operation. func (c *FlinkClient) TriggerSavepoint( - apiBaseURL string, jobID string, dir string) (SavepointTriggerID, error) { + apiBaseURL string, jobID string, dir string, cancel bool) (SavepointTriggerID, error) { var url = fmt.Sprintf("%s/jobs/%s/savepoints", apiBaseURL, jobID) var jsonStr = fmt.Sprintf(`{ "target-directory" : "%s", - "cancel-job" : false - }`, dir) + "cancel-job" : %v + }`, dir, cancel) var triggerID = SavepointTriggerID{} var err = c.HTTPClient.Post(url, []byte(jsonStr), &triggerID) return triggerID, err } +// TakeSavepoint takes savepoint, blocks until it succeeds or fails. +func (c *FlinkClient) TakeSavepoint( + apiBaseURL string, jobID string, dir string) (SavepointStatus, error) { + var triggerID = SavepointTriggerID{} + var status = SavepointStatus{JobID: jobID} + var err error + + triggerID, err = c.TriggerSavepoint(apiBaseURL, jobID, dir, false) + if err != nil { + return SavepointStatus{}, err + } + + for i := 0; i < 12; i++ { + status, err = c.GetSavepointStatus(apiBaseURL, jobID, triggerID.RequestID) + if err == nil && status.Completed { + return status, nil + } + time.Sleep(5 * time.Second) + } + + return status, err +} + // GetSavepointStatus returns savepoint status. // // Flink API response examples: @@ -181,39 +204,3 @@ func (c *FlinkClient) GetSavepointStatus( } return status, err } - -// TakeSavepoint takes savepoint, blocks until it suceeds or fails. -func (c *FlinkClient) TakeSavepoint( - apiBaseURL string, jobID string, dir string) (SavepointStatus, error) { - var triggerID = SavepointTriggerID{} - var status = SavepointStatus{JobID: jobID} - var err error - - triggerID, err = c.TriggerSavepoint(apiBaseURL, jobID, dir) - if err != nil { - return SavepointStatus{}, err - } - - for i := 0; i < 12; i++ { - status, err = c.GetSavepointStatus(apiBaseURL, jobID, triggerID.RequestID) - if err == nil && status.Completed { - return status, nil - } - time.Sleep(5 * time.Second) - } - - return status, err -} - -func (c *FlinkClient) TakeSavepointAsync( - apiBaseURL string, jobID string, dir string) (string, error) { - var triggerID = SavepointTriggerID{} - var err error - - triggerID, err = c.TriggerSavepoint(apiBaseURL, jobID, dir) - if err != nil { - return "", err - } - - return triggerID.RequestID, err -} diff --git a/controllers/flinkcluster_converter.go b/controllers/flinkcluster_converter.go index 1fd8d8c4..fa474313 100644 --- a/controllers/flinkcluster_converter.go +++ b/controllers/flinkcluster_converter.go @@ -102,7 +102,7 @@ func getDesiredJobManagerStatefulSet( var jobManagerStatefulSetName = getJobManagerStatefulSetName(clusterName) var podLabels = getComponentLabels(*flinkCluster, "jobmanager") podLabels = mergeLabels(podLabels, jobManagerSpec.PodLabels) - var statefulSetLabels = mergeLabels(podLabels, getRevisionHashLabels(flinkCluster.Status)) + var statefulSetLabels = mergeLabels(podLabels, getRevisionHashLabels(&flinkCluster.Status.Revision)) var securityContext = jobManagerSpec.SecurityContext // Make Volume, VolumeMount to use configMap data for flink-conf.yaml, if flinkProperties is provided. var volumes []corev1.Volume @@ -263,7 +263,7 @@ func getDesiredJobManagerService( var jobManagerServiceName = getJobManagerServiceName(clusterName) var podLabels = getComponentLabels(*flinkCluster, "jobmanager") podLabels = mergeLabels(podLabels, jobManagerSpec.PodLabels) - var serviceLabels = mergeLabels(podLabels, getRevisionHashLabels(flinkCluster.Status)) + var serviceLabels = mergeLabels(podLabels, getRevisionHashLabels(&flinkCluster.Status.Revision)) var jobManagerService = &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: clusterNamespace, @@ -325,7 +325,7 @@ func getDesiredJobManagerIngress( var ingressTLS []extensionsv1beta1.IngressTLS var labels = mergeLabels( getComponentLabels(*flinkCluster, "jobmanager"), - getRevisionHashLabels(flinkCluster.Status)) + getRevisionHashLabels(&flinkCluster.Status.Revision)) if jobManagerIngressSpec.HostFormat != nil { ingressHost = getJobManagerIngressHost(*jobManagerIngressSpec.HostFormat, clusterName) } @@ -400,7 +400,7 @@ func getDesiredTaskManagerStatefulSet( var taskManagerStatefulSetName = getTaskManagerStatefulSetName(clusterName) var podLabels = getComponentLabels(*flinkCluster, "taskmanager") podLabels = mergeLabels(podLabels, taskManagerSpec.PodLabels) - var statefulSetLabels = mergeLabels(podLabels, getRevisionHashLabels(flinkCluster.Status)) + var statefulSetLabels = mergeLabels(podLabels, getRevisionHashLabels(&flinkCluster.Status.Revision)) var securityContext = taskManagerSpec.SecurityContext @@ -549,7 +549,7 @@ func getDesiredConfigMap( var configMapName = getConfigMapName(clusterName) var labels = mergeLabels( getClusterLabels(*flinkCluster), - getRevisionHashLabels(flinkCluster.Status)) + getRevisionHashLabels(&flinkCluster.Status.Revision)) var flinkHeapSize = calFlinkHeapSize(flinkCluster) // Properties which should be provided from real deployed environment. var flinkProps = map[string]string{ @@ -594,19 +594,18 @@ func getDesiredConfigMap( // Gets the desired job spec from a cluster spec. func getDesiredJob(observed *ObservedClusterState) *batchv1.Job { var flinkCluster = observed.cluster + var recorded = flinkCluster.Status var jobSpec = flinkCluster.Spec.Job - var jobStatus = flinkCluster.Status.Components.Job + var jobStatus = recorded.Components.Job if jobSpec == nil { return nil } - // Unless update has been triggered or the job needs to be restarted, keep the job to be stopped in that state. - if !(isUpdateTriggered(flinkCluster.Status) || shouldRestartJob(jobSpec.RestartPolicy, jobStatus)) { - // Job cancel requested or stopped already - if isJobCancelRequested(*flinkCluster) || isJobStopped(jobStatus) { - return nil - } + // When the job should be stopped, keep that state unless update is triggered or the job must to be restarted. + if (shouldStopJob(flinkCluster) || isJobStopped(jobStatus)) && + !(observed.isClusterUpdating() || shouldRestartJob(jobSpec, jobStatus)) { + return nil } var clusterSpec = flinkCluster.Spec @@ -621,14 +620,14 @@ func getDesiredJob(observed *ObservedClusterState) *batchv1.Job { "%s:%d", jobManagerServiceName, *jobManagerSpec.Ports.UI) var podLabels = getClusterLabels(*flinkCluster) podLabels = mergeLabels(podLabels, jobManagerSpec.PodLabels) - var jobLabels = mergeLabels(podLabels, getRevisionHashLabels(flinkCluster.Status)) + var jobLabels = mergeLabels(podLabels, getRevisionHashLabels(&recorded.Revision)) var jobArgs = []string{"bash", "/opt/flink-operator/submit-job.sh"} jobArgs = append(jobArgs, "--jobmanager", jobManagerAddress) if jobSpec.ClassName != nil { jobArgs = append(jobArgs, "--class", *jobSpec.ClassName) } - var fromSavepoint = convertFromSavepoint(jobSpec, flinkCluster.Status.Components.Job) + var fromSavepoint = convertFromSavepoint(jobSpec, jobStatus, &recorded.Revision) if fromSavepoint != nil { jobArgs = append(jobArgs, "--fromSavepoint", *fromSavepoint) } @@ -771,15 +770,16 @@ func getDesiredJob(observed *ObservedClusterState) *batchv1.Job { // Flink job will be restored from the latest savepoint created by the operator. // // case 3) When latest created savepoint is unavailable, use the savepoint from which current job was restored. -func convertFromSavepoint(jobSpec *v1beta1.JobSpec, jobStatus *v1beta1.JobStatus) *string { +func convertFromSavepoint(jobSpec *v1beta1.JobSpec, jobStatus *v1beta1.JobStatus, revision *v1beta1.RevisionStatus) *string { switch { // Creating for the first time case jobStatus == nil: - if jobSpec.FromSavepoint != nil && *jobSpec.FromSavepoint != "" { + if !isBlank(jobSpec.FromSavepoint) { return jobSpec.FromSavepoint } + return nil // Updating with FromSavepoint provided - case jobStatus.State == v1beta1.JobStateUpdating && jobSpec.FromSavepoint != nil && *jobSpec.FromSavepoint != "": + case isUpdateTriggered(revision) && !isBlank(jobSpec.FromSavepoint): return jobSpec.FromSavepoint // Latest savepoint case jobStatus.SavepointLocation != "": @@ -872,7 +872,7 @@ func shouldCleanup( return false } - if isUpdateTriggered(cluster.Status) { + if isUpdateTriggered(&cluster.Status.Revision) { return false } @@ -1070,9 +1070,9 @@ func getComponentLabels(cluster v1beta1.FlinkCluster, component string) map[stri }) } -func getRevisionHashLabels(status v1beta1.FlinkClusterStatus) map[string]string { +func getRevisionHashLabels(r *v1beta1.RevisionStatus) map[string]string { return map[string]string{ - RevisionNameLabel: getNextRevisionName(status), + RevisionNameLabel: getNextRevisionName(r), } } diff --git a/controllers/flinkcluster_converter_test.go b/controllers/flinkcluster_converter_test.go index 4bcae053..c881d63b 100644 --- a/controllers/flinkcluster_converter_test.go +++ b/controllers/flinkcluster_converter_test.go @@ -273,7 +273,7 @@ func TestGetDesiredClusterState(t *testing.T) { }, }, Status: v1beta1.FlinkClusterStatus{ - NextRevision: "flinkjobcluster-sample-85dc8f749-1", + Revision: v1beta1.RevisionStatus{NextRevision: "flinkjobcluster-sample-85dc8f749-1"}, }, }, } @@ -984,7 +984,7 @@ func TestSecurityContext(t *testing.T) { }, }, Status: v1beta1.FlinkClusterStatus{ - NextRevision: "flinkjobcluster-sample-85dc8f749-1", + Revision: v1beta1.RevisionStatus{NextRevision: "flinkjobcluster-sample-85dc8f749-1"}, }, }, } @@ -1022,7 +1022,7 @@ func TestSecurityContext(t *testing.T) { }, }, Status: v1beta1.FlinkClusterStatus{ - NextRevision: "flinkjobcluster-sample-85dc8f749-1", + Revision: v1beta1.RevisionStatus{NextRevision: "flinkjobcluster-sample-85dc8f749-1"}, }, }, } diff --git a/controllers/flinkcluster_observer.go b/controllers/flinkcluster_observer.go index b6abf1c1..04ebfa9c 100644 --- a/controllers/flinkcluster_observer.go +++ b/controllers/flinkcluster_observer.go @@ -24,6 +24,7 @@ import ( v1beta1 "github.com/googlecloudplatform/flink-operator/api/v1beta1" "github.com/googlecloudplatform/flink-operator/controllers/flinkclient" "github.com/googlecloudplatform/flink-operator/controllers/history" + yaml "gopkg.in/yaml.v2" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -55,29 +56,65 @@ type ObservedClusterState struct { jmService *corev1.Service jmIngress *extensionsv1beta1.Ingress tmStatefulSet *appsv1.StatefulSet - job *batchv1.Job - jobPod *corev1.Pod - flinkJobStatus FlinkJobStatus - flinkJobSubmitLog *FlinkJobSubmitLog - savepoint *Savepoint - revisionStatus *RevisionStatus + flinkJob FlinkJob + flinkJobSubmitter FlinkJobSubmitter + savepoint Savepoint + revision Revision observeTime time.Time + updateState UpdateState } -type FlinkJobStatus struct { - flinkJob *flinkclient.JobStatus - flinkJobList *flinkclient.JobStatusList - flinkJobsUnexpected []string +type FlinkJob struct { + status *flinkclient.JobStatus + list *flinkclient.JobStatusList + unexpected []string } -type FlinkJobSubmitLog struct { +type FlinkJobSubmitter struct { + job *batchv1.Job + pod *corev1.Pod + log *SubmitterLog +} + +type SubmitterLog struct { JobID string `yaml:"jobID,omitempty"` Message string `yaml:"message"` } type Savepoint struct { - *flinkclient.SavepointStatus - savepointErr error + status *flinkclient.SavepointStatus + error error +} + +type Revision struct { + currentRevision *appsv1.ControllerRevision + nextRevision *appsv1.ControllerRevision + collisionCount int32 +} + +func (o *ObservedClusterState) isClusterUpdating() bool { + return o.updateState == UpdateStateInProgress +} + +func (r *Revision) isUpdateTriggered() bool { + return getRevisionWithNameNumber(r.currentRevision) != getRevisionWithNameNumber(r.nextRevision) +} + +// Job submitter status. +func (s *FlinkJobSubmitter) getState() JobSubmitState { + switch { + case s.job == nil: + break + case s.job.Status.Succeeded == 0 && s.job.Status.Failed == 0: + return JobDeployStateInProgress + case s.job.Status.Failed > 0: + return JobDeployStateFailed + case s.job.Status.Succeeded > 0: + if s.log != nil && s.log.JobID != "" { + return JobDeployStateSucceeded + } + } + return JobDeployStateUnknown } // Observes the state of the cluster and its components. @@ -196,86 +233,122 @@ func (observer *ClusterStateObserver) observe( } // (Optional) Savepoint. - // Savepoint observe error do not affect deploy reconciliation loop. - observer.observeSavepoint(observed) + var observedSavepoint Savepoint + err = observer.observeSavepoint(observed.cluster, &observedSavepoint) + if err != nil { + log.Error(err, "Failed to get Flink job savepoint status") + } else { + log.Info("Observed Flink job savepoint status", "status", observedSavepoint.status) + } + observed.savepoint = observedSavepoint // (Optional) job. err = observer.observeJob(observed) + if err != nil { + log.Error(err, "Failed to get Flink job status") + return err + } observed.observeTime = time.Now() + observed.updateState = getUpdateState(observed) - return err + return nil } func (observer *ClusterStateObserver) observeJob( observed *ObservedClusterState) error { - if observed.cluster == nil { + // Either the cluster has been deleted or it is a session cluster. + if observed.cluster == nil || observed.cluster.Spec.Job == nil { return nil } + var log = observer.log + var recorded = observed.cluster.Status + var err error - // Observe following - var observedJob *batchv1.Job - var observedFlinkJobStatus FlinkJobStatus - var observedFlinkJobSubmitLog *FlinkJobSubmitLog + // Observe the Flink job submitter. + var submitter FlinkJobSubmitter + err = observer.observeSubmitter(&submitter) + if err != nil { + log.Error(err, "Failed to get the status of the job submitter") + } + observed.flinkJobSubmitter = submitter - var recordedJobStatus = observed.cluster.Status.Components.Job - var err error + // Observe the Flink job status. + var flinkJobID string + // Get the ID from the job submitter. + if submitter.log != nil && submitter.log.JobID != "" { + flinkJobID = submitter.log.JobID + } else + // Or get the job ID from the recorded job status which is written in previous iteration. + if recorded.Components.Job != nil { + flinkJobID = recorded.Components.Job.ID + } + var observedFlinkJob FlinkJob + observer.observeFlinkJobStatus(observed, flinkJobID, &observedFlinkJob) + observed.flinkJob = observedFlinkJob + + return nil +} + +func (observer *ClusterStateObserver) observeSubmitter(submitter *FlinkJobSubmitter) error { var log = observer.log + var err error - // Either the cluster has been deleted or it is a session cluster. - if observed.cluster == nil || observed.cluster.Spec.Job == nil { - return nil - } + // Observe following + var job *batchv1.Job + var pod *corev1.Pod + var podLog *SubmitterLog // Job resource. - observedJob = new(batchv1.Job) - err = observer.observeJobResource(observedJob) + job = new(batchv1.Job) + err = observer.observeSubmitterJob(job) if err != nil { if client.IgnoreNotFound(err) != nil { - log.Error(err, "Failed to get job") + log.Error(err, "Failed to get the submitter job") return err } - log.Info("Observed job submitter", "state", "nil") - observedJob = nil + log.Info("Observed submitter job", "state", "nil") + job = nil } else { - log.Info("Observed job submitter", "state", *observedJob) + log.Info("Observed submitter job", "state", *job) } - observed.job = observedJob - - // Get Flink job ID. - // While job state is pending and job submitter is completed, extract the job ID from the pod termination log. - var jobSubmitCompleted = observedJob != nil && (observedJob.Status.Succeeded > 0 || observedJob.Status.Failed > 0) - var jobInPendingState = recordedJobStatus != nil && recordedJobStatus.State == v1beta1.JobStatePending - var flinkJobID string - if jobSubmitCompleted && jobInPendingState { - var observedJobPod *corev1.Pod - - // Get job submitter pod resource. - observedJobPod = new(corev1.Pod) - err = observer.observeJobPod(observedJobPod) - if err != nil { - log.Error(err, "Failed to get job pod") - } - observed.jobPod = observedJobPod + submitter.job = job - // Extract submit result. - observedFlinkJobSubmitLog, err = getFlinkJobSubmitLog(observedJobPod) - if err != nil { - log.Error(err, "Failed to extract job submit result") - } - if observedFlinkJobSubmitLog != nil && observedFlinkJobSubmitLog.JobID != "" { - flinkJobID = observedFlinkJobSubmitLog.JobID - } - observed.flinkJobSubmitLog = observedFlinkJobSubmitLog + // Get the job submission log. + // When the recorded job state is pending or updating, and the actual submission is completed, + // extract the job submission log from the pod termination log. + if submitter.job == nil { + return nil } - // Or get the job ID from the recorded job status which is written previous iteration. - if flinkJobID == "" && recordedJobStatus != nil { - flinkJobID = recordedJobStatus.ID + // Get job submitter pod resource. + pod = new(corev1.Pod) + err = observer.observeJobSubmitterPod(pod) + if err != nil { + log.Error(err, "Failed to get the submitter pod") + return err + } else if pod == nil { + log.Info("Observed submitter job pod", "state", "nil") + return nil + } else { + log.Info("Observed submitter job pod", "state", *pod) } + submitter.pod = pod - // Flink job status. - observer.observeFlinkJobStatus(observed, flinkJobID, &observedFlinkJobStatus) - observed.flinkJobStatus = observedFlinkJobStatus + // Extract submit result. + var jobSubmissionCompleted = job.Status.Succeeded > 0 || job.Status.Failed > 0 + if !jobSubmissionCompleted { + return nil + } + log.Info("Extracting the result of job submission because it is completed") + podLog = new(SubmitterLog) + err = observeFlinkJobSubmitterLog(pod, podLog) + if err != nil { + log.Error(err, "Failed to extract the job submission result") + podLog = nil + } else { + log.Info("Observed submitter log", "state", *podLog) + } + submitter.log = podLog return nil } @@ -288,11 +361,11 @@ func (observer *ClusterStateObserver) observeJob( func (observer *ClusterStateObserver) observeFlinkJobStatus( observed *ObservedClusterState, flinkJobID string, - flinkJobStatus *FlinkJobStatus) { + flinkJob *FlinkJob) { var log = observer.log // Observe following - var flinkJob *flinkclient.JobStatus + var flinkJobStatus *flinkclient.JobStatus var flinkJobList *flinkclient.JobStatusList var flinkJobsUnexpected []string @@ -314,10 +387,7 @@ func (observer *ClusterStateObserver) observeFlinkJobStatus( log.Info("Failed to get Flink job status list.", "error", err) return } - log.Info("Observed Flink job status list", "jobs", flinkJobList.Jobs) - - // Initialize flinkJobStatus if flink API is available. - flinkJobStatus.flinkJobList = flinkJobList + flinkJob.list = flinkJobList // Extract the current job status and unexpected jobs, if submitted job ID is provided. if flinkJobID == "" { @@ -325,14 +395,13 @@ func (observer *ClusterStateObserver) observeFlinkJobStatus( } for _, job := range flinkJobList.Jobs { if flinkJobID == job.ID { - flinkJob = new(flinkclient.JobStatus) - *flinkJob = job + flinkJobStatus = &job } else if getFlinkJobDeploymentState(job.Status) == v1beta1.JobStateRunning { flinkJobsUnexpected = append(flinkJobsUnexpected, job.ID) } } - flinkJobStatus.flinkJob = flinkJob - flinkJobStatus.flinkJobsUnexpected = flinkJobsUnexpected + flinkJob.status = flinkJobStatus + flinkJob.unexpected = flinkJobsUnexpected // It is okay if there are multiple jobs, but at most one of them is // expected to be running. This is typically caused by job client @@ -345,40 +414,30 @@ func (observer *ClusterStateObserver) observeFlinkJobStatus( "", "unexpected jobs", flinkJobsUnexpected) } if flinkJob != nil { - log.Info("Observed Flink job", "flink job", *flinkJob) + log.Info("Observed Flink job", "flink job", flinkJob) } return } -func (observer *ClusterStateObserver) observeSavepoint(observed *ObservedClusterState) error { - var log = observer.log - - if observed.cluster == nil { +func (observer *ClusterStateObserver) observeSavepoint(cluster *v1beta1.FlinkCluster, savepoint *Savepoint) error { + if cluster == nil || + cluster.Status.Savepoint == nil || + cluster.Status.Savepoint.State != v1beta1.SavepointStateInProgress { return nil } // Get savepoint status in progress. - var savepointStatus = observed.cluster.Status.Savepoint - if savepointStatus != nil && savepointStatus.State == v1beta1.SavepointStateInProgress { - var flinkAPIBaseURL = getFlinkAPIBaseURL(observed.cluster) - var jobID = savepointStatus.JobID - var triggerID = savepointStatus.TriggerID - var savepoint flinkclient.SavepointStatus - var err error - - savepoint, err = observer.flinkClient.GetSavepointStatus(flinkAPIBaseURL, jobID, triggerID) - observed.savepoint = &Savepoint{SavepointStatus: &savepoint} - if err == nil && len(savepoint.FailureCause.StackTrace) > 0 { - err = fmt.Errorf("%s", savepoint.FailureCause.StackTrace) - } - if err != nil { - observed.savepoint.savepointErr = err - log.Info("Failed to get savepoint.", "error", err, "jobID", jobID, "triggerID", triggerID) - } - return err - } - return nil + var flinkAPIBaseURL = getFlinkAPIBaseURL(cluster) + var recordedSavepoint = cluster.Status.Savepoint + var jobID = recordedSavepoint.JobID + var triggerID = recordedSavepoint.TriggerID + + savepointStatus, err := observer.flinkClient.GetSavepointStatus(flinkAPIBaseURL, jobID, triggerID) + savepoint.status = &savepointStatus + savepoint.error = err + + return err } func (observer *ClusterStateObserver) observeCluster( @@ -483,7 +542,7 @@ func (observer *ClusterStateObserver) observeJobManagerIngress( observedIngress) } -func (observer *ClusterStateObserver) observeJobResource( +func (observer *ClusterStateObserver) observeSubmitterJob( observedJob *batchv1.Job) error { var clusterNamespace = observer.request.Namespace var clusterName = observer.request.Name @@ -497,10 +556,9 @@ func (observer *ClusterStateObserver) observeJobResource( observedJob) } -// observeJobPod observes job submitter pod. -func (observer *ClusterStateObserver) observeJobPod( +// observeJobSubmitterPod observes job submitter pod. +func (observer *ClusterStateObserver) observeJobSubmitterPod( observedPod *corev1.Pod) error { - var log = observer.log var clusterNamespace = observer.request.Namespace var clusterName = observer.request.Name var podSelector = labels.SelectorFromSet(map[string]string{"job-name": getJobName(clusterName)}) @@ -512,25 +570,15 @@ func (observer *ClusterStateObserver) observeJobPod( client.InNamespace(clusterNamespace), client.MatchingLabelsSelector{Selector: podSelector}) if err != nil { - if client.IgnoreNotFound(err) != nil { - log.Error(err, "Failed to get job submitter pod list") - return err - } - log.Info("Observed job submitter pod list", "state", "nil") - } else { - log.Info("Observed job submitter pod list", "state", *podList) + return err } - - if podList != nil && len(podList.Items) > 0 { + if len(podList.Items) == 0 { + observedPod = nil + } else { podList.Items[0].DeepCopyInto(observedPod) } - return nil -} -type RevisionStatus struct { - currentRevision *appsv1.ControllerRevision - nextRevision *appsv1.ControllerRevision - collisionCount int32 + return nil } // syncRevisionStatus synchronizes current FlinkCluster resource and its child ControllerRevision resources. @@ -548,20 +596,19 @@ func (observer *ClusterStateObserver) syncRevisionStatus(observed *ObservedClust return nil } - var revisions = observed.revisions var cluster = observed.cluster - var recordedStatus = cluster.Status + var revisions = observed.revisions + var recorded = cluster.Status var currentRevision, nextRevision *appsv1.ControllerRevision var controllerHistory = observer.history - var revisionStatus = observed.revisionStatus revisionCount := len(revisions) history.SortControllerRevisions(revisions) // Use a local copy of cluster.Status.CollisionCount to avoid modifying cluster.Status directly. var collisionCount int32 - if recordedStatus.CollisionCount != nil { - collisionCount = *recordedStatus.CollisionCount + if recorded.Revision.CollisionCount != nil { + collisionCount = *recorded.Revision.CollisionCount } // create a new revision from the current cluster @@ -594,12 +641,12 @@ func (observer *ClusterStateObserver) syncRevisionStatus(observed *ObservedClust } // if the current revision is nil we initialize the history by setting it to the next revision - if recordedStatus.CurrentRevision == "" { + if recorded.Revision.CurrentRevision == "" { currentRevision = nextRevision // attempt to find the revision that corresponds to the current revision } else { for i := range revisions { - if revisions[i].Name == getCurrentRevisionName(recordedStatus) { + if revisions[i].Name == getCurrentRevisionName(&recorded.Revision) { currentRevision = revisions[i] break } @@ -609,12 +656,12 @@ func (observer *ClusterStateObserver) syncRevisionStatus(observed *ObservedClust return fmt.Errorf("current ControlRevision resoucre not found") } - // update revision status - revisionStatus = new(RevisionStatus) - revisionStatus.currentRevision = currentRevision.DeepCopy() - revisionStatus.nextRevision = nextRevision.DeepCopy() - revisionStatus.collisionCount = collisionCount - observed.revisionStatus = revisionStatus + // Update revision status. + observed.revision = Revision{ + currentRevision: currentRevision.DeepCopy(), + nextRevision: nextRevision.DeepCopy(), + collisionCount: collisionCount, + } // maintain the revision history limit err = observer.truncateHistory(observed) @@ -646,3 +693,22 @@ func (observer *ClusterStateObserver) truncateHistory(observed *ObservedClusterS } return nil } + +// observeFlinkJobSubmit extract submit result from the pod termination log. +func observeFlinkJobSubmitterLog(observedPod *corev1.Pod, submitterLog *SubmitterLog) error { + var containerStatuses = observedPod.Status.ContainerStatuses + if len(containerStatuses) == 0 || + containerStatuses[0].State.Terminated == nil || + containerStatuses[0].State.Terminated.Message == "" { + return fmt.Errorf("job pod found, but no termination log") + } + + // The job submission script writes the submission log to the pod termination log at the end of execution. + // If the job submission is successful, the extracted job ID is also included. + // The job submit script writes the submission result in YAML format, + // so parse it here to get the ID - if available - and log. + // Note: https://kubernetes.io/docs/tasks/debug-application-cluster/determine-reason-pod-failure/ + var rawJobSubmissionResult = containerStatuses[0].State.Terminated.Message + var err = yaml.Unmarshal([]byte(rawJobSubmissionResult), submitterLog) + return err +} diff --git a/controllers/flinkcluster_reconciler.go b/controllers/flinkcluster_reconciler.go index 7763b04b..5d37ba55 100644 --- a/controllers/flinkcluster_reconciler.go +++ b/controllers/flinkcluster_reconciler.go @@ -65,7 +65,7 @@ func (reconciler *ClusterReconciler) reconcile() (ctrl.Result, error) { return ctrl.Result{}, nil } - if getUpdateState(reconciler.observed) == UpdateStateInProgress { + if reconciler.observed.isClusterUpdating() { reconciler.log.Info("The cluster update is in progress") } // If batch-scheduling enabled @@ -137,7 +137,8 @@ func (reconciler *ClusterReconciler) reconcileStatefulSet( } if desiredStatefulSet != nil && observedStatefulSet != nil { - if getUpdateState(reconciler.observed) == UpdateStateInProgress { + var cluster = reconciler.observed.cluster + if reconciler.observed.isClusterUpdating() && !isComponentUpdated(observedStatefulSet, cluster) { updateComponent := fmt.Sprintf("%v StatefulSet", component) var err error if *reconciler.observed.cluster.Spec.RecreateOnUpdate { @@ -150,7 +151,7 @@ func (reconciler *ClusterReconciler) reconcileStatefulSet( } return nil } - log.Info("Statefulset already exists, no action") + log.Info("StatefulSet already exists, no action") return nil } @@ -179,7 +180,7 @@ func (reconciler *ClusterReconciler) createStatefulSet( func (reconciler *ClusterReconciler) deleteOldComponent(desired runtime.Object, observed runtime.Object, component string) error { var log = reconciler.log.WithValues("component", component) - if isComponentUpdated(observed, *reconciler.observed.cluster) { + if isComponentUpdated(observed, reconciler.observed.cluster) { reconciler.log.Info(fmt.Sprintf("%v is already updated, no action", component)) return nil } @@ -253,7 +254,8 @@ func (reconciler *ClusterReconciler) reconcileJobManagerService() error { } if desiredJmService != nil && observedJmService != nil { - if getUpdateState(reconciler.observed) == UpdateStateInProgress { + var cluster = reconciler.observed.cluster + if reconciler.observed.isClusterUpdating() && !isComponentUpdated(observedJmService, cluster) { // v1.Service API does not handle update correctly when below values are empty. desiredJmService.SetResourceVersion(observedJmService.GetResourceVersion()) desiredJmService.Spec.ClusterIP = observedJmService.Spec.ClusterIP @@ -321,7 +323,8 @@ func (reconciler *ClusterReconciler) reconcileJobManagerIngress() error { } if desiredJmIngress != nil && observedJmIngress != nil { - if getUpdateState(reconciler.observed) == UpdateStateInProgress { + var cluster = reconciler.observed.cluster + if reconciler.observed.isClusterUpdating() && !isComponentUpdated(observedJmIngress, cluster) { var err error if *reconciler.observed.cluster.Spec.RecreateOnUpdate { err = reconciler.deleteOldComponent(desiredJmIngress, observedJmIngress, "JobManager ingress") @@ -386,7 +389,8 @@ func (reconciler *ClusterReconciler) reconcileConfigMap() error { } if desiredConfigMap != nil && observedConfigMap != nil { - if getUpdateState(reconciler.observed) == UpdateStateInProgress { + var cluster = reconciler.observed.cluster + if reconciler.observed.isClusterUpdating() && !isComponentUpdated(observedConfigMap, cluster) { var err error if *reconciler.observed.cluster.Spec.RecreateOnUpdate { err = reconciler.deleteOldComponent(desiredConfigMap, observedConfigMap, "ConfigMap") @@ -446,8 +450,9 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { var log = reconciler.log var desiredJob = reconciler.desired.Job var observed = reconciler.observed - var observedJob = observed.job - var recordedJobStatus = observed.cluster.Status.Components.Job + var recorded = observed.cluster.Status + var jobSpec = observed.cluster.Spec.Job + var jobStatus = recorded.Components.Job var activeFlinkJob bool var err error var jobID = reconciler.getFlinkJobID() @@ -457,51 +462,43 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { var newControlStatus *v1beta1.FlinkClusterControlStatus defer reconciler.updateStatus(&newSavepointStatus, &newControlStatus) - // Cancel unexpected jobs - if len(observed.flinkJobStatus.flinkJobsUnexpected) > 0 { - log.Info("Cancelling unexpected running job(s)") - err = reconciler.cancelUnexpectedJobs(false /* takeSavepoint */) - return requeueResult, err - } - // Check if Flink job is active - if isJobActive(recordedJobStatus) { + if isJobActive(jobStatus) { activeFlinkJob = true - - // Trigger savepoint if required. - if len(jobID) > 0 { - savepointTriggerReason := reconciler.shouldTakeSavepoint() - if savepointTriggerReason != "" { - newSavepointStatus, _ = reconciler.takeSavepointAsync(jobID, savepointTriggerReason) - if userControl := getNewUserControlRequest(observed.cluster); userControl == v1beta1.ControlNameSavepoint { - newControlStatus = getUserControlStatus(userControl, v1beta1.ControlStateInProgress) - } - } - } } else { activeFlinkJob = false } - // Create Flink job submitter + // Create new Flink job submitter when starting new job, updating job or restarting job in failure. if desiredJob != nil && !activeFlinkJob { - // If update triggered, wait until all Flink cluster components are replaced with next revision. - if !isClusterUpdateToDate(observed) { - return requeueResult, nil + log.Info("Deploying Flink job") + + var unexpectedJobs = observed.flinkJob.unexpected + if len(unexpectedJobs) > 0 { + if jobSpec.MaxStateAgeToRestoreSeconds != nil { + log.Info("Cancelling unexpected running job(s)") + err = reconciler.cancelUnexpectedJobs(false /* takeSavepoint */) + return requeueResult, err + } + // In this case the user should identify the cause of the problem + // so that the job is not accidentally executed multiple times by mistake or the Flink operator's error. + err = fmt.Errorf("failed to create job submitter, unexpected jobs found: %v", unexpectedJobs) + return ctrl.Result{}, err } // Create Flink job submitter log.Info("Updating job status to proceed creating new job submitter") - err = reconciler.updateStatusForNewJob() + // Job status must be updated before creating a job submitter to ensure the observed job is the job submitted by the operator. + err = reconciler.updateJobDeployStatus() if err != nil { - log.Info("Not proceed to create new job submitter because job status update failed") + log.Info("Failed to update the job status for job submission") return requeueResult, err } - log.Info("Creating new job submitter") - if observedJob != nil { - log.Info("Deleting old job submitter") - err = reconciler.deleteJob(observedJob) + var observedSubmitter = observed.flinkJobSubmitter.job + if observedSubmitter != nil { + log.Info("Found old job submitter") + err = reconciler.deleteJob(observedSubmitter) if err != nil { - log.Info("Failed to delete previous job submitter") return requeueResult, err } } @@ -511,27 +508,31 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { } if desiredJob != nil && activeFlinkJob { - var restartPolicy = observed.cluster.Spec.Job.RestartPolicy - var recordedJobStatus = observed.cluster.Status.Components.Job - - // Stop Flink job for update or recovery. - var stopReason string - if shouldUpdateJob(observed) { - stopReason = "update" - // Change job state. - err := reconciler.changeJobStateToUpdating() - if err != nil { - log.Error(err, "Failed to change job status for update", "error", err) - return requeueResult, err + if jobStatus.State == v1beta1.JobStateDeploying { + log.Info("Job submitter is deployed and wait until it is completed") + return requeueResult, nil + } + if isUpdateTriggered(&recorded.Revision) { + log.Info("Preparing job update") + var shouldSuspend = jobSpec.TakeSavepointOnUpdate == nil || *jobSpec.TakeSavepointOnUpdate == true || isBlank(jobSpec.FromSavepoint) + if shouldSuspend { + newSavepointStatus, err = reconciler.trySuspendJob() + } else { + err = reconciler.cancelJob() } - } else if shouldRestartJob(restartPolicy, recordedJobStatus) { - stopReason = "recovery" + return requeueResult, err } - if stopReason != "" { - log.Info(fmt.Sprintf("Restart job for %s.", stopReason)) - err := reconciler.restartJob() - if err != nil { - log.Info("Failed to restart job.") + + // Trigger savepoint if required. + if len(jobID) > 0 { + var savepointTriggerReason = reconciler.shouldTakeSavepoint() + if savepointTriggerReason != "" { + newSavepointStatus, err = reconciler.triggerSavepoint(jobID, savepointTriggerReason, false) + } + // Get new control status when the savepoint reason matches the requested control. + var userControl = getNewControlRequest(observed.cluster) + if userControl == v1beta1.ControlNameSavepoint && savepointTriggerReason == v1beta1.SavepointTriggerReasonUserRequested { + newControlStatus = getControlStatus(userControl, v1beta1.ControlStateInProgress) } return requeueResult, err } @@ -543,33 +544,15 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { // Job cancel requested. Stop Flink job. if desiredJob == nil && activeFlinkJob { log.Info("Stopping job", "jobID", jobID) - log.Info("Check savepoint status first.") - var savepoint *v1beta1.SavepointStatus - if newSavepointStatus != nil { - savepoint = newSavepointStatus - } else { - savepoint = observed.cluster.Status.Savepoint - } - // If savepoint is taken, proceed to cancel job. - var err = reconciler.cancelFlinkJobAsync(jobID, savepoint) - // When there is an error, get failed status to record. - if err != nil { - log.Error(err, "Failed to cancel job", "jobID", jobID) - newControlStatus = getFailedCancelStatus(err) - return requeueResult, err + newSavepointStatus, err = reconciler.trySuspendJob() + var userControl = getNewControlRequest(observed.cluster) + if userControl == v1beta1.ControlNameJobCancel { + newControlStatus = getControlStatus(userControl, v1beta1.ControlStateInProgress) } - if userControl := getNewUserControlRequest(observed.cluster); userControl == v1beta1.ControlNameJobCancel { - newControlStatus = getUserControlStatus(userControl, v1beta1.ControlStateInProgress) - } - // When savepoint is in progress yet, check it in next iteration. - if savepoint != nil && savepoint.State == v1beta1.SavepointStateInProgress { - return requeueResult, nil - } - log.Info("Successfully job cancelled.") - return requeueResult, nil + return requeueResult, err } - if isJobStopped(recordedJobStatus) { + if isJobStopped(jobStatus) { log.Info("Job has finished, no action") } @@ -581,12 +564,12 @@ func (reconciler *ClusterReconciler) createJob(job *batchv1.Job) error { var log = reconciler.log var k8sClient = reconciler.k8sClient - log.Info("Submitting job", "resource", *job) + log.Info("Creating job submitter", "resource", *job) var err = k8sClient.Create(context, job) if err != nil { - log.Info("Failed to created job", "error", err) + log.Info("Failed to created job submitter", "error", err) } else { - log.Info("Job created") + log.Info("Job submitter created") } return err } @@ -599,13 +582,13 @@ func (reconciler *ClusterReconciler) deleteJob(job *batchv1.Job) error { var deletePolicy = metav1.DeletePropagationBackground var deleteOption = client.DeleteOptions{PropagationPolicy: &deletePolicy} - log.Info("Deleting job", "job", job) + log.Info("Deleting job submitter", "job", job) var err = k8sClient.Delete(context, job, &deleteOption) err = client.IgnoreNotFound(err) if err != nil { - log.Error(err, "Failed to delete job") + log.Error(err, "Failed to delete job submitter") } else { - log.Info("Job deleted") + log.Info("Job submitter deleted") } return err } @@ -618,36 +601,55 @@ func (reconciler *ClusterReconciler) getFlinkJobID() string { return "" } -func (reconciler *ClusterReconciler) restartJob() error { +func (reconciler *ClusterReconciler) trySuspendJob() (*v1beta1.SavepointStatus, error) { + var log = reconciler.log + var recorded = reconciler.observed.cluster.Status + var jobID = reconciler.getFlinkJobID() + + log.Info("Checking the conditions for progressing") + var canSuspend = reconciler.canSuspendJob(jobID, recorded.Savepoint) + if canSuspend { + log.Info("Triggering savepoint for suspending job") + var newSavepointStatus, err = reconciler.triggerSavepoint(jobID, v1beta1.SavepointTriggerReasonUpdate, true) + if err != nil { + log.Info("Failed to trigger savepoint", "jobID", jobID, "triggerID", newSavepointStatus.TriggerID, "error", err) + } else { + log.Info("Successfully savepoint triggered", "jobID", jobID, "triggerID", newSavepointStatus.TriggerID) + } + return newSavepointStatus, err + } + + return nil, nil +} + +func (reconciler *ClusterReconciler) cancelJob() error { var log = reconciler.log - var observedJob = reconciler.observed.job - var observedFlinkJob = reconciler.observed.flinkJobStatus.flinkJob + var observedFlinkJob = reconciler.observed.flinkJob.status - log.Info("Stopping Flink job to restart", "", observedFlinkJob) + log.Info("Stopping Flink job", "", observedFlinkJob) var err = reconciler.cancelRunningJobs(false /* takeSavepoint */) if err != nil { + log.Info("Failed to stop Flink job") return err } - if observedJob != nil { - var err = reconciler.deleteJob(observedJob) + // TODO: It would be nice not to delete the job submitters immediately, and retain the latest ones for debug. + var observedSubmitter = reconciler.observed.flinkJobSubmitter.job + if observedSubmitter != nil { + var err = reconciler.deleteJob(observedSubmitter) if err != nil { log.Error( - err, "Failed to delete job submitter", "job", observedJob) + err, "Failed to delete job submitter", "job", observedSubmitter) return err } } - // Do not create new job immediately, leave it to the next reconciliation, - // because we still need to be able to create the new job if we encounter - // ephemeral error here. It is better to organize the logic in a central place. - return nil } func (reconciler *ClusterReconciler) cancelUnexpectedJobs( takeSavepoint bool) error { - var unexpectedJobs = reconciler.observed.flinkJobStatus.flinkJobsUnexpected + var unexpectedJobs = reconciler.observed.flinkJob.unexpected return reconciler.cancelJobs(takeSavepoint, unexpectedJobs) } @@ -655,8 +657,8 @@ func (reconciler *ClusterReconciler) cancelUnexpectedJobs( func (reconciler *ClusterReconciler) cancelRunningJobs( takeSavepoint bool) error { var log = reconciler.log - var runningJobs = reconciler.observed.flinkJobStatus.flinkJobsUnexpected - var flinkJob = reconciler.observed.flinkJobStatus.flinkJob + var runningJobs = reconciler.observed.flinkJob.unexpected + var flinkJob = reconciler.observed.flinkJob.status if flinkJob != nil && flinkJob.ID != "" && getFlinkJobDeploymentState(flinkJob.Status) == v1beta1.JobStateRunning { runningJobs = append(runningJobs, flinkJob.ID) @@ -688,12 +690,11 @@ func (reconciler *ClusterReconciler) cancelJobs( func (reconciler *ClusterReconciler) cancelFlinkJob(jobID string, takeSavepoint bool) error { var log = reconciler.log if takeSavepoint && canTakeSavepoint(*reconciler.observed.cluster) { + log.Info("Taking savepoint before stopping job", "jobID", jobID) var err = reconciler.takeSavepoint(jobID) if err != nil { return err } - } else { - log.Info("Skip taking savepoint before stopping job", "jobID", jobID) } var apiBaseURL = getFlinkAPIBaseURL(reconciler.observed.cluster) @@ -701,123 +702,103 @@ func (reconciler *ClusterReconciler) cancelFlinkJob(jobID string, takeSavepoint return reconciler.flinkClient.StopJob(apiBaseURL, jobID) } -// Trigger savepoint if it is possible, then return the savepoint status to update. -// When savepoint was already triggered, return the current observed status. -// If savepoint cannot be triggered, taking savepoint is skipped, or the triggered savepoint is completed, proceed to stop the job. -func (reconciler *ClusterReconciler) cancelFlinkJobAsync(jobID string, savepoint *v1beta1.SavepointStatus) error { +// canSuspendJob +func (reconciler *ClusterReconciler) canSuspendJob(jobID string, s *v1beta1.SavepointStatus) bool { var log = reconciler.log - var cluster = reconciler.observed.cluster - var err error + var firstTry = !finalSavepointRequested(jobID, s) + if firstTry { + return true + } - switch savepoint.State { + switch s.State { case v1beta1.SavepointStateSucceeded: - log.Info("Successfully savepoint created. Proceed to stop job.") + log.Info("Successfully savepoint completed, wait until the job stops") + return false case v1beta1.SavepointStateInProgress: - log.Info("Triggered savepoint already, wait until it is completed.") - return nil + log.Info("Savepoint is in progress, wait until it is completed") + return false case v1beta1.SavepointStateTriggerFailed: - log.Info("Failed to trigger savepoint.") - return fmt.Errorf("failed to trigger savepoint: %v", *savepoint) - // Cannot be reached here with these states, because job-cancel control should be finished with failed savepoint states in updater. + log.Info("Savepoint trigger failed in previous request") case v1beta1.SavepointStateFailed: - fallthrough - default: - return fmt.Errorf("unexpected savepoint status: %v", *savepoint) + log.Info("Savepoint failed on previous request") } - var apiBaseURL = getFlinkAPIBaseURL(cluster) - log.Info("Stopping job", "jobID", jobID) - err = reconciler.flinkClient.StopJob(apiBaseURL, jobID) - if err != nil { - return fmt.Errorf("failed to stop job: %v", err) + var retryTimeArrived = hasTimeElapsed(s.UpdateTime, time.Now(), SavepointRequestRetryIntervalSec) + if !retryTimeArrived { + log.Info("Wait until next retry time arrived") } - return nil + return retryTimeArrived } func (reconciler *ClusterReconciler) shouldTakeSavepoint() string { - var log = reconciler.log var observed = reconciler.observed var cluster = observed.cluster var jobSpec = observed.cluster.Spec.Job var jobStatus = observed.cluster.Status.Components.Job - var savepointStatus = observed.cluster.Status.Savepoint - var userControl = getNewUserControlRequest(cluster) + var newRequestedControl = getNewControlRequest(cluster) if !canTakeSavepoint(*reconciler.observed.cluster) { return "" } + // Savepoint trigger priority is user request including update and job stop. switch { - // Triggered by user requested savepoint - case userControl == v1beta1.ControlNameSavepoint: - return v1beta1.SavepointTriggerReasonUserRequested - // Triggered by user requested job cancel - case userControl == v1beta1.ControlNameJobCancel: + // TODO: spec.job.cancelRequested will be deprecated + // Should stop job with savepoint by user requested control + case newRequestedControl == v1beta1.ControlNameJobCancel || (jobSpec.CancelRequested != nil && *jobSpec.CancelRequested): return v1beta1.SavepointTriggerReasonJobCancel - // Triggered by update - case getUpdateState(observed) == UpdateStatePreparing: - // TODO: apply exponential backoff retry - // If failed to take savepoint, retry after SavepointRequestRetryIntervalSec. - var takeSavepointOnUpdate = jobSpec.TakeSavepointOnUpdate == nil || *jobSpec.TakeSavepointOnUpdate == true - if takeSavepointOnUpdate && - !isSavepointUpToDate(observed.observeTime, jobSpec, jobStatus) && // savepoint up to date - (savepointStatus == nil || savepointStatus.State != v1beta1.SavepointStateInProgress) && // no savepoint in progress - hasTimeElapsed(savepointStatus.UpdateTime, time.Now(), SavepointRequestRetryIntervalSec) { // retry interval arrived - return v1beta1.SavepointTriggerReasonUpdate - } - // Triggered by schedule (auto savepoint) + // TODO: spec.job.savepointGeneration will be deprecated + // Take savepoint by user request + case newRequestedControl == v1beta1.ControlNameSavepoint: + fallthrough + case jobSpec.SavepointGeneration > jobStatus.SavepointGeneration: + // Triggered by savepointGeneration increased. + // When previous savepoint is failed, savepoint trigger by spec.job.savepointGeneration is not possible + // because the field cannot be increased more. + // Note: checkSavepointGeneration in flinkcluster_validate.go + return v1beta1.SavepointTriggerReasonUserRequested + // Scheduled auto savepoint case jobSpec.AutoSavepointSeconds != nil: // Check if next trigger time arrived. var compareTime string - if len(jobStatus.LastSavepointTime) == 0 { + if len(jobStatus.SavepointTime) == 0 { compareTime = jobStatus.StartTime } else { - compareTime = jobStatus.LastSavepointTime + compareTime = jobStatus.SavepointTime } var nextTime = getTimeAfterAddedSeconds(compareTime, int64(*jobSpec.AutoSavepointSeconds)) if time.Now().After(nextTime) { return v1beta1.SavepointTriggerReasonScheduled } - // TODO: spec.job.savepointGeneration will be deprecated - // Triggered by savepointGeneration increased - // When previous savepoint is failed, savepoint trigger by spec.job.savepointGeneration is not possible - // because the field cannot be increased more by validator. - // Note: checkSavepointGeneration in flinkcluster_validate.go - case jobSpec.SavepointGeneration > jobStatus.SavepointGeneration: - log.Info( - "Savepoint is requested", - "statusGen", jobStatus.SavepointGeneration, - "specGen", jobSpec.SavepointGeneration) - return v1beta1.SavepointTriggerReasonUserRequested } return "" } // Trigger savepoint for a job then return savepoint status to update. -func (reconciler *ClusterReconciler) takeSavepointAsync(jobID string, triggerReason string) (*v1beta1.SavepointStatus, error) { +func (reconciler *ClusterReconciler) triggerSavepoint(jobID string, triggerReason string, cancel bool) (*v1beta1.SavepointStatus, error) { var log = reconciler.log var cluster = reconciler.observed.cluster var apiBaseURL = getFlinkAPIBaseURL(reconciler.observed.cluster) var triggerSuccess bool - var triggerID string + var triggerID flinkclient.SavepointTriggerID var message string var err error log.Info(fmt.Sprintf("Trigger savepoint for %s", triggerReason), "jobID", jobID) - triggerID, err = reconciler.flinkClient.TakeSavepointAsync(apiBaseURL, jobID, *cluster.Spec.Job.SavepointsDir) + triggerID, err = reconciler.flinkClient.TriggerSavepoint(apiBaseURL, jobID, *cluster.Spec.Job.SavepointsDir, cancel) if err != nil { // limit message size to 1KiB if message = err.Error(); len(message) > 1024 { message = message[:1024] + "..." } triggerSuccess = false - log.Info("Savepoint trigger is failed.", "jobID", jobID, "triggerID", triggerID, "error", err) + log.Info("Failed to trigger savepoint", "jobID", jobID, "triggerID", triggerID, "error", err) } else { triggerSuccess = true - log.Info("Savepoint is triggered successfully.", "jobID", jobID, "triggerID", triggerID) + log.Info("Successfully savepoint triggered", "jobID", jobID, "triggerID", triggerID) } - newSavepointStatus := getNewSavepointStatus(jobID, triggerID, triggerReason, message, triggerSuccess) + newSavepointStatus := reconciler.getNewSavepointStatus(triggerID.RequestID, triggerReason, message, triggerSuccess) return newSavepointStatus, err } @@ -895,73 +876,56 @@ func (reconciler *ClusterReconciler) updateStatus(ss **v1beta1.SavepointStatus, } } -func (reconciler *ClusterReconciler) updateStatusForNewJob() error { +func (reconciler *ClusterReconciler) updateJobDeployStatus() error { var log = reconciler.log - var newJobStatus *v1beta1.JobStatus - var desiredJob = reconciler.desired.Job - var clusterClone = reconciler.observed.cluster.DeepCopy() + var observedCluster = reconciler.observed.cluster + var desiredJobSubmitter = reconciler.desired.Job var err error - if clusterClone.Status.Components.Job != nil { - newJobStatus = clusterClone.Status.Components.Job - switch previousJobState := newJobStatus.State; previousJobState { - case v1beta1.JobStateFailed: - newJobStatus.RestartCount++ - case v1beta1.JobStateUpdating: - newJobStatus.RestartCount = 0 - } - } else { - newJobStatus = &v1beta1.JobStatus{} - clusterClone.Status.Components.Job = newJobStatus - } - var fromSavepoint = getFromSavepoint(desiredJob.Spec) - newJobStatus.ID = "" - newJobStatus.State = v1beta1.JobStatePending - newJobStatus.FromSavepoint = fromSavepoint - if newJobStatus.SavepointLocation != "" { - // Latest savepoint should be "fromSavepoint" - newJobStatus.SavepointLocation = fromSavepoint + var clusterClone = observedCluster.DeepCopy() + var newJob = clusterClone.Status.Components.Job + + // Latest savepoint location should be fromSavepoint. + var fromSavepoint = getFromSavepoint(desiredJobSubmitter.Spec) + newJob.FromSavepoint = fromSavepoint + if newJob.SavepointLocation != "" { + newJob.SavepointLocation = fromSavepoint } + setTimestamp(&newJob.DeployTime) // Mark as job submitter is deployed. setTimestamp(&clusterClone.Status.LastUpdateTime) err = reconciler.k8sClient.Status().Update(reconciler.context, clusterClone) if err != nil { log.Error( - err, "Failed to update job status for new job submission", "error", err) + err, "Failed to update job status for new job submitter", "error", err) } else { - log.Info("Succeeded to update job status for new job submission.", "job status", newJobStatus) + log.Info("Succeeded to update job status for new job submitter.", "job status", newJob) } return err } -func (reconciler *ClusterReconciler) changeJobStateToUpdating() error { - var clusterClone = reconciler.observed.cluster.DeepCopy() - var newJobStatus = clusterClone.Status.Components.Job - newJobStatus.ID = "" - newJobStatus.State = v1beta1.JobStateUpdating - setTimestamp(&clusterClone.Status.LastUpdateTime) - err := reconciler.k8sClient.Status().Update(reconciler.context, clusterClone) - return err -} - -// If job cancellation is failed, fill the status message with error message. -// Then, the state will be transited to the failed by the updater. -func getFailedCancelStatus(cancelErr error) *v1beta1.FlinkClusterControlStatus { - var state string - var message string +// getNewSavepointStatus returns newly triggered savepoint status. +func (reconciler *ClusterReconciler) getNewSavepointStatus(triggerID string, triggerReason string, message string, triggerSuccess bool) *v1beta1.SavepointStatus { + var jobID = reconciler.getFlinkJobID() + var savepointState string var now string setTimestamp(&now) - state = v1beta1.ControlStateInProgress - // limit message size to 1KiB - if message = cancelErr.Error(); len(message) > 1024 { - message = message[:1024] + "..." - } - return &v1beta1.FlinkClusterControlStatus{ - Name: v1beta1.ControlNameJobCancel, - State: state, - UpdateTime: now, - Message: message, - } + + if triggerSuccess { + savepointState = v1beta1.SavepointStateInProgress + } else { + savepointState = v1beta1.SavepointStateTriggerFailed + } + var savepointStatus = &v1beta1.SavepointStatus{ + JobID: jobID, + TriggerID: triggerID, + TriggerReason: triggerReason, + TriggerTime: now, + UpdateTime: now, + Message: message, + State: savepointState, + } + return savepointStatus } // Convert raw time to object and add `addedSeconds` to it, diff --git a/controllers/flinkcluster_updater.go b/controllers/flinkcluster_updater.go index 43cdf172..0e15fe76 100644 --- a/controllers/flinkcluster_updater.go +++ b/controllers/flinkcluster_updater.go @@ -22,9 +22,7 @@ package controllers import ( "context" "encoding/json" - "errors" "fmt" - "github.com/googlecloudplatform/flink-operator/controllers/flinkclient" "k8s.io/apimachinery/pkg/types" "reflect" "time" @@ -186,6 +184,7 @@ func (updater *ClusterStatusUpdater) createStatusChangeEvent( } } +// TODO: Need to organize func (updater *ClusterStatusUpdater) deriveClusterStatus( recorded *v1beta1.FlinkClusterStatus, observed *ObservedClusterState) v1beta1.FlinkClusterStatus { @@ -193,13 +192,10 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( var runningComponents = 0 // jmStatefulSet, jmService, tmStatefulSet. var totalComponents = 3 - var updateState = getUpdateState(*observed) - var isClusterUpdating = !isClusterUpdateToDate(*observed) && updateState == UpdateStateInProgress - var isJobUpdating = recorded.Components.Job != nil && recorded.Components.Job.State == v1beta1.JobStateUpdating // ConfigMap. var observedConfigMap = observed.configMap - if !isComponentUpdated(observedConfigMap, *observed.cluster) && isJobUpdating { + if !isComponentUpdated(observedConfigMap, observed.cluster) && observed.isClusterUpdating() { recorded.Components.ConfigMap.DeepCopyInto(&status.Components.ConfigMap) status.Components.ConfigMap.State = v1beta1.ComponentStateUpdating } else if observedConfigMap != nil { @@ -215,7 +211,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( // JobManager StatefulSet. var observedJmStatefulSet = observed.jmStatefulSet - if !isComponentUpdated(observedJmStatefulSet, *observed.cluster) && isJobUpdating { + if !isComponentUpdated(observedJmStatefulSet, observed.cluster) && observed.isClusterUpdating() { recorded.Components.JobManagerStatefulSet.DeepCopyInto(&status.Components.JobManagerStatefulSet) status.Components.JobManagerStatefulSet.State = v1beta1.ComponentStateUpdating } else if observedJmStatefulSet != nil { @@ -234,7 +230,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( // JobManager service. var observedJmService = observed.jmService - if !isComponentUpdated(observedJmService, *observed.cluster) && isJobUpdating { + if !isComponentUpdated(observedJmService, observed.cluster) && observed.isClusterUpdating() { recorded.Components.JobManagerService.DeepCopyInto(&status.Components.JobManagerService) status.Components.JobManagerService.State = v1beta1.ComponentStateUpdating } else if observedJmService != nil { @@ -285,7 +281,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( // (Optional) JobManager ingress. var observedJmIngress = observed.jmIngress - if !isComponentUpdated(observedJmIngress, *observed.cluster) && isJobUpdating { + if !isComponentUpdated(observedJmIngress, observed.cluster) && observed.isClusterUpdating() { status.Components.JobManagerIngress = &v1beta1.JobManagerIngressStatus{} recorded.Components.JobManagerIngress.DeepCopyInto(status.Components.JobManagerIngress) status.Components.JobManagerIngress.State = v1beta1.ComponentStateUpdating @@ -368,7 +364,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( // TaskManager StatefulSet. var observedTmStatefulSet = observed.tmStatefulSet - if !isComponentUpdated(observedTmStatefulSet, *observed.cluster) && isJobUpdating { + if !isComponentUpdated(observedTmStatefulSet, observed.cluster) && observed.isClusterUpdating() { recorded.Components.TaskManagerStatefulSet.DeepCopyInto(&status.Components.TaskManagerStatefulSet) status.Components.TaskManagerStatefulSet.State = v1beta1.ComponentStateUpdating } else if observedTmStatefulSet != nil { @@ -388,10 +384,6 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( } } - // (Optional) Job. - // Update job status. - status.Components.Job = updater.getJobStatus() - // Derive the new cluster state. switch recorded.State { case "", v1beta1.ClusterStateCreating: @@ -401,10 +393,10 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( status.State = v1beta1.ClusterStateRunning } case v1beta1.ClusterStateUpdating: - if isClusterUpdating { + if observed.isClusterUpdating() { status.State = v1beta1.ClusterStateUpdating } else if runningComponents < totalComponents { - if isUpdateTriggered(*recorded) { + if isUpdateTriggered(&recorded.Revision) { status.State = v1beta1.ClusterStateUpdating } else { status.State = v1beta1.ClusterStateReconciling @@ -414,10 +406,10 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( } case v1beta1.ClusterStateRunning, v1beta1.ClusterStateReconciling: - var jobStatus = status.Components.Job - if isClusterUpdating { + var jobStatus = recorded.Components.Job + if observed.isClusterUpdating() { status.State = v1beta1.ClusterStateUpdating - } else if isJobStopped(jobStatus) { + } else if !isUpdateTriggered(&recorded.Revision) && isJobStopped(jobStatus) { var policy = observed.cluster.Spec.Job.CleanupPolicy if jobStatus.State == v1beta1.JobStateSucceeded && policy.AfterJobSucceeds != v1beta1.CleanupActionKeepCluster { @@ -438,7 +430,8 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( } case v1beta1.ClusterStateStopping, v1beta1.ClusterStatePartiallyStopped: - if isClusterUpdating { + //if isClusterUpdating { + if observed.isClusterUpdating() { status.State = v1beta1.ClusterStateUpdating } else if runningComponents == 0 { status.State = v1beta1.ClusterStateStopped @@ -448,7 +441,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( status.State = v1beta1.ClusterStateStopping } case v1beta1.ClusterStateStopped: - if isUpdateTriggered(*recorded) { + if isUpdateTriggered(&recorded.Revision) { status.State = v1beta1.ClusterStateUpdating } else { status.State = v1beta1.ClusterStateStopped @@ -457,35 +450,33 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( panic(fmt.Sprintf("Unknown cluster state: %v", recorded.State)) } + // (Optional) Job. + // Update job status. + status.Components.Job = updater.deriveJobStatus() + + // (Optional) Savepoint. // Update savepoint status if it is in progress or requested. - status.Savepoint = updater.getUpdatedSavepointStatus( - observed.savepoint, + var newJobStatus = status.Components.Job + status.Savepoint = updater.deriveSavepointStatus( + &observed.savepoint, recorded.Savepoint, - recorded.Components.Job, + newJobStatus, updater.getFlinkJobID()) - // User requested control - status.Control = getControlStatus(observed.cluster, &status, recorded) - - // Update finished - if updateState == UpdateStateFinished { - status.CurrentRevision = observed.cluster.Status.NextRevision - updater.log.Info("Finished update.") - } - - // Update revision status - status.NextRevision = getRevisionWithNameNumber(observed.revisionStatus.nextRevision) - if status.CurrentRevision == "" { - if recorded.CurrentRevision == "" { - status.CurrentRevision = getRevisionWithNameNumber(observed.revisionStatus.currentRevision) - } else { - status.CurrentRevision = recorded.CurrentRevision - } - } - if observed.revisionStatus.collisionCount != 0 { - status.CollisionCount = new(int32) - *status.CollisionCount = observed.revisionStatus.collisionCount - } + // (Optional) Control. + // Update user requested control status. + status.Control = deriveControlStatus( + observed.cluster, + status.Savepoint, + status.Components.Job, + recorded.Control) + + // Update revision status. + // When update completed, finish the process by marking CurrentRevision to NextRevision. + status.Revision = deriveRevisionStatus( + observed.updateState, + &observed.revision, + &recorded.Revision) return status } @@ -496,16 +487,16 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( // to transient error or being skiped as an optimization. // If this returned nil, it is the state that job is not submitted or not identified yet. func (updater *ClusterStatusUpdater) getFlinkJobID() *string { - // Observed from active job manager - var observedFlinkJob = updater.observed.flinkJobStatus.flinkJob + // Observed from active job manager. + var observedFlinkJob = updater.observed.flinkJob.status if observedFlinkJob != nil && len(observedFlinkJob.ID) > 0 { return &observedFlinkJob.ID } - // Observed from job submitter (when job manager is not ready yet) - var observedJobSubmitLog = updater.observed.flinkJobSubmitLog - if observedJobSubmitLog != nil && observedJobSubmitLog.JobID != "" { - return &observedJobSubmitLog.JobID + // Observed from job submitter (when Flink API is not ready). + var observedJobSubmitterLog = updater.observed.flinkJobSubmitter.log + if observedJobSubmitterLog != nil && observedJobSubmitterLog.JobID != "" { + return &observedJobSubmitterLog.JobID } // Recorded. @@ -517,98 +508,125 @@ func (updater *ClusterStatusUpdater) getFlinkJobID() *string { return nil } -func (updater *ClusterStatusUpdater) getJobStatus() *v1beta1.JobStatus { +func (updater *ClusterStatusUpdater) deriveJobStatus() *v1beta1.JobStatus { var observed = updater.observed - var observedJob = updater.observed.job - var observedFlinkJob = updater.observed.flinkJobStatus.flinkJob - var observedCluster = updater.observed.cluster - var observedSavepoint = updater.observed.savepoint - var recordedJobStatus = updater.observed.cluster.Status.Components.Job - var newJobStatus *v1beta1.JobStatus - - if recordedJobStatus == nil { + var observedCluster = observed.cluster + var jobSpec = observedCluster.Spec.Job + if jobSpec == nil { return nil } - newJobStatus = recordedJobStatus.DeepCopy() - // Derive job state - var jobState string + var observedSubmitter = observed.flinkJobSubmitter + var observedFlinkJob = observed.flinkJob.status + var observedSavepoint = observed.savepoint + var recorded = observedCluster.Status + var savepoint = recorded.Savepoint + var oldJob = recorded.Components.Job + var newJob *v1beta1.JobStatus + + // Derive new job state. + if oldJob != nil { + newJob = oldJob.DeepCopy() + } else { + newJob = new(v1beta1.JobStatus) + } + var newJobState string + var newJobID string switch { - // When updating stopped job - case isUpdateTriggered(observedCluster.Status) && isJobStopped(recordedJobStatus): - jobState = v1beta1.JobStateUpdating - // Already terminated state - case isJobTerminated(observedCluster.Spec.Job.RestartPolicy, recordedJobStatus): - jobState = recordedJobStatus.State - // Derive state from the observed Flink job + case oldJob == nil: + newJobState = v1beta1.JobStatePending + case observed.isClusterUpdating(): + newJobState = v1beta1.JobStateUpdating + case shouldRestartJob(jobSpec, oldJob): + newJobState = v1beta1.JobStateRestarting + case isJobPending(oldJob) && oldJob.DeployTime != "": + newJobState = v1beta1.JobStateDeploying + case isJobStopped(oldJob): + newJobState = oldJob.State + // Derive the job state from the observed Flink job, if it exists. case observedFlinkJob != nil: - jobState = getFlinkJobDeploymentState(observedFlinkJob.Status) - if jobState == "" { - updater.log.Error(errors.New("failed to get Flink job deployment state"), "observed flink job status", observedFlinkJob.Status) - jobState = recordedJobStatus.State - } - // When Flink job not found - case isFlinkAPIReady(observed): - switch recordedJobStatus.State { - case v1beta1.JobStateRunning: - jobState = v1beta1.JobStateLost - case v1beta1.JobStatePending: - // Flink job is submitted but not confirmed via job manager yet - var jobSubmitSucceeded = updater.getFlinkJobID() != nil - // Flink job submit is in progress - var jobSubmitInProgress = observedJob != nil && - observedJob.Status.Succeeded == 0 && - observedJob.Status.Failed == 0 - if jobSubmitSucceeded || jobSubmitInProgress { - jobState = v1beta1.JobStatePending - break - } - jobState = v1beta1.JobStateFailed - default: - jobState = recordedJobStatus.State + newJobState = getFlinkJobDeploymentState(observedFlinkJob.Status) + // Unexpected Flink job state + if newJobState == "" { + panic(fmt.Sprintf("Unknown Flink job status: %s", observedFlinkJob.Status)) } - // When Flink API unavailable + newJobID = observedFlinkJob.ID + // When Flink job not found in JobManager or JobManager is unavailable + case isFlinkAPIReady(observed.flinkJob.list): + if oldJob.State == v1beta1.JobStateRunning { + newJobState = v1beta1.JobStateLost + break + } + fallthrough default: - if recordedJobStatus.State == v1beta1.JobStatePending { - var jobSubmitFailed = observedJob != nil && observedJob.Status.Failed > 0 - if jobSubmitFailed { - jobState = v1beta1.JobStateFailed - break - } + if oldJob.State != v1beta1.JobStateDeploying { + newJobState = oldJob.State + break + } + // Job submitter is deployed but tracking failed. + var submitterState = observedSubmitter.getState() + if submitterState == JobDeployStateUnknown { + newJobState = v1beta1.JobStateLost + // Case in which the job submission clearly fails even if it is not confirmed by JobManager + // Job submitter is deployed but failed. + } else if submitterState == JobDeployStateFailed { + newJobState = v1beta1.JobStateDeployFailed } - jobState = recordedJobStatus.State + newJobState = oldJob.State } - - // Flink Job ID - if jobState == v1beta1.JobStateUpdating { - newJobStatus.ID = "" - } else if observedFlinkJob != nil { - newJobStatus.ID = observedFlinkJob.ID + newJob.State = newJobState + if newJobID != "" { + newJob.ID = newJobID } - // State - newJobStatus.State = jobState - - // Flink job start time - // TODO: It would be nice to set StartTime with the timestamp retrieved from the Flink job API like /jobs/{job-id}. - if jobState == v1beta1.JobStateRunning && newJobStatus.StartTime == "" { - setTimestamp(&newJobStatus.StartTime) + // Derived new job status if the state is changed. + if oldJob == nil || oldJob.State != newJob.State { + // TODO: It would be ideal to set the times with the timestamp retrieved from the Flink API like /jobs/{job-id}. + switch { + case isJobPending(newJob): + newJob.DeployTime = "" + if newJob.State == v1beta1.JobStateUpdating { + newJob.RestartCount = 0 + } else if newJob.State == v1beta1.JobStateRestarting { + newJob.RestartCount++ + } + case newJob.State == v1beta1.JobStateDeploying: + newJob.ID = "" + newJob.StartTime = "" + newJob.EndTime = "" + case newJob.State == v1beta1.JobStateRunning: + setTimestamp(&newJob.StartTime) + newJob.EndTime = "" + // When job started, the savepoint is not the final state of the job any more. + if oldJob.FinalSavepoint { + newJob.FinalSavepoint = false + } + case isJobStopped(newJob): + if newJob.EndTime == "" { + setTimestamp(&newJob.EndTime) + } + // When tracking failed, we cannot guarantee if the savepoint is the final job state. + if newJob.State == v1beta1.JobStateLost && oldJob.FinalSavepoint { + newJob.FinalSavepoint = false + } + } } // Savepoint - if newJobStatus != nil && observedSavepoint != nil && observedSavepoint.IsSuccessful() { - newJobStatus.SavepointGeneration++ - newJobStatus.LastSavepointTriggerID = observedSavepoint.TriggerID - newJobStatus.SavepointLocation = observedSavepoint.Location - - // TODO: LastSavepointTime should be set with the timestamp generated in job manager. + if observedSavepoint.status != nil && observedSavepoint.status.IsSuccessful() { + newJob.SavepointGeneration++ + newJob.SavepointLocation = observedSavepoint.status.Location + if finalSavepointRequested(newJob.ID, savepoint) { + newJob.FinalSavepoint = true + } + // TODO: SavepointTime should be set with the timestamp generated in job manager. // Currently savepoint complete timestamp is not included in savepoints API response. // Whereas checkpoint API returns the timestamp latest_ack_timestamp. // Note: https://ci.apache.org/projects/flink/flink-docs-stable/ops/rest_api.html#jobs-jobid-checkpoints-details-checkpointid - setTimestamp(&newJobStatus.LastSavepointTime) + setTimestamp(&newJob.SavepointTime) } - return newJobStatus + return newJob } func (updater *ClusterStatusUpdater) isStatusChanged( @@ -725,15 +743,17 @@ func (updater *ClusterStatusUpdater) isStatusChanged( newStatus.Savepoint) changed = true } - if newStatus.CurrentRevision != currentStatus.CurrentRevision || - newStatus.NextRevision != currentStatus.NextRevision || - (newStatus.CollisionCount != nil && currentStatus.CollisionCount == nil) || - (currentStatus.CollisionCount != nil && *newStatus.CollisionCount != *currentStatus.CollisionCount) { + var nr = newStatus.Revision // New revision status + var cr = currentStatus.Revision // Current revision status + if nr.CurrentRevision != cr.CurrentRevision || + nr.NextRevision != cr.NextRevision || + (nr.CollisionCount != nil && cr.CollisionCount == nil) || + (cr.CollisionCount != nil && *nr.CollisionCount != *cr.CollisionCount) { updater.log.Info( "FlinkCluster revision status changed", "current", - fmt.Sprintf("currentRevision: %v, nextRevision: %v, collisionCount: %v", currentStatus.CurrentRevision, currentStatus.NextRevision, currentStatus.CollisionCount), + fmt.Sprintf("currentRevision: %v, nextRevision: %v, collisionCount: %v", cr.CurrentRevision, cr.NextRevision, cr.CollisionCount), "new", - fmt.Sprintf("currentRevision: %v, nextRevision: %v, collisionCount: %v", newStatus.CurrentRevision, newStatus.NextRevision, newStatus.CollisionCount)) + fmt.Sprintf("currentRevision: %v, nextRevision: %v, collisionCount: %v", nr.CurrentRevision, nr.NextRevision, nr.CollisionCount)) changed = true } return changed @@ -776,46 +796,51 @@ func (updater *ClusterStatusUpdater) clearControlAnnotation(newControlStatus *v1 return nil } -func (updater *ClusterStatusUpdater) getUpdatedSavepointStatus( +func (updater *ClusterStatusUpdater) deriveSavepointStatus( observedSavepoint *Savepoint, recordedSavepointStatus *v1beta1.SavepointStatus, - recordedJobStatus *v1beta1.JobStatus, + newJobStatus *v1beta1.JobStatus, flinkJobID *string) *v1beta1.SavepointStatus { if recordedSavepointStatus == nil { return nil } - var savepointStatus = recordedSavepointStatus.DeepCopy() + // Derived savepoint status to return + var s = recordedSavepointStatus.DeepCopy() var errMsg string - if savepointStatus.State == v1beta1.SavepointStateInProgress && observedSavepoint != nil { + + // Update the savepoint status when observed savepoint is found. + if s.State == v1beta1.SavepointStateInProgress && observedSavepoint != nil { switch { - case observedSavepoint.IsSuccessful(): - savepointStatus.State = v1beta1.SavepointStateSucceeded - case observedSavepoint.IsFailed(): - savepointStatus.State = v1beta1.SavepointStateFailed - errMsg = fmt.Sprintf("Savepoint error: %v", observedSavepoint.FailureCause.StackTrace) - case observedSavepoint.savepointErr != nil: - if err, ok := observedSavepoint.savepointErr.(*flinkclient.HTTPError); ok { - savepointStatus.State = v1beta1.SavepointStateFailed - errMsg = fmt.Sprintf("Failed to get savepoint status: %v", err) - } + case observedSavepoint.status.IsSuccessful(): + s.State = v1beta1.SavepointStateSucceeded + case observedSavepoint.status.IsFailed(): + s.State = v1beta1.SavepointStateFailed + errMsg = fmt.Sprintf("Savepoint error: %v", observedSavepoint.status.FailureCause.StackTrace) + case observedSavepoint.error != nil: + s.State = v1beta1.SavepointStateFailed + errMsg = fmt.Sprintf("Failed to get savepoint status: %v", observedSavepoint.error) } } - if savepointStatus.State == v1beta1.SavepointStateInProgress { + + // Check failure conditions of savepoint in progress. + if s.State == v1beta1.SavepointStateInProgress { switch { - case isJobStopped(recordedJobStatus): + case isJobStopped(newJobStatus): errMsg = "Flink job is stopped." - savepointStatus.State = v1beta1.SavepointStateFailed + s.State = v1beta1.SavepointStateFailed case flinkJobID == nil: errMsg = "Flink job is not identified." - savepointStatus.State = v1beta1.SavepointStateFailed + s.State = v1beta1.SavepointStateFailed case flinkJobID != nil && (recordedSavepointStatus.TriggerID != "" && *flinkJobID != recordedSavepointStatus.JobID): errMsg = "Savepoint triggered Flink job is lost." - savepointStatus.State = v1beta1.SavepointStateFailed + s.State = v1beta1.SavepointStateFailed } } + + // Make up message. if errMsg != "" { - if savepointStatus.TriggerReason == v1beta1.SavepointTriggerReasonUpdate { + if s.TriggerReason == v1beta1.SavepointTriggerReasonUpdate { errMsg = "Failed to take savepoint for update. " + "The update process is being postponed until a savepoint is available. " + errMsg @@ -823,75 +848,96 @@ func (updater *ClusterStatusUpdater) getUpdatedSavepointStatus( if len(errMsg) > 1024 { errMsg = errMsg[:1024] } - savepointStatus.Message = errMsg + s.Message = errMsg } - return savepointStatus + + return s } -func getControlStatus(cluster *v1beta1.FlinkCluster, - new *v1beta1.FlinkClusterStatus, - recorded *v1beta1.FlinkClusterStatus) *v1beta1.FlinkClusterControlStatus { - var userControl = cluster.Annotations[v1beta1.ControlAnnotation] - var controlStatus *v1beta1.FlinkClusterControlStatus - var controlRequest = getNewUserControlRequest(cluster) +func deriveControlStatus( + cluster *v1beta1.FlinkCluster, + newSavepoint *v1beta1.SavepointStatus, + newJob *v1beta1.JobStatus, + recordedControl *v1beta1.FlinkClusterControlStatus) *v1beta1.FlinkClusterControlStatus { + var controlRequest = getNewControlRequest(cluster) + + // Derived control status to return + var c *v1beta1.FlinkClusterControlStatus // New control status if controlRequest != "" { - controlStatus = getUserControlStatus(controlRequest, v1beta1.ControlStateRequested) - return controlStatus + c = getControlStatus(controlRequest, v1beta1.ControlStateRequested) + return c } - // Update control status in progress - if recorded.Control != nil && userControl == recorded.Control.Name && - recorded.Control.State == v1beta1.ControlStateInProgress { - controlStatus = recorded.Control.DeepCopy() - switch recorded.Control.Name { + // Update control status in progress. + if recordedControl != nil && recordedControl.State == v1beta1.ControlStateInProgress { + c = recordedControl.DeepCopy() + switch recordedControl.Name { case v1beta1.ControlNameJobCancel: - if new.Components.Job.State == v1beta1.JobStateCancelled { - controlStatus.State = v1beta1.ControlStateSucceeded - setTimestamp(&controlStatus.UpdateTime) - } else if isJobTerminated(cluster.Spec.Job.RestartPolicy, recorded.Components.Job) { - controlStatus.Message = "Aborted job cancellation: job is terminated already." - controlStatus.State = v1beta1.ControlStateFailed - setTimestamp(&controlStatus.UpdateTime) - } else if new.Savepoint != nil && new.Savepoint.State == v1beta1.SavepointStateFailed { - controlStatus.Message = "Aborted job cancellation: failed to take savepoint." - controlStatus.State = v1beta1.ControlStateFailed - setTimestamp(&controlStatus.UpdateTime) - } else if recorded.Control.Message != "" { - controlStatus.State = v1beta1.ControlStateFailed - setTimestamp(&controlStatus.UpdateTime) + if newSavepoint.State == v1beta1.SavepointStateSucceeded && newJob.State == v1beta1.JobStateCancelled { + c.State = v1beta1.ControlStateSucceeded + } else if isJobStopped(newJob) { + c.Message = "Aborted job cancellation: savepoint is not completed, but job is stopped already." + c.State = v1beta1.ControlStateFailed + } else if newSavepoint.TriggerReason == v1beta1.SavepointTriggerReasonJobCancel && + (newSavepoint.State == v1beta1.SavepointStateFailed || newSavepoint.State == v1beta1.SavepointStateTriggerFailed) { + c.Message = "Aborted job cancellation: failed to take savepoint." + c.State = v1beta1.ControlStateFailed } case v1beta1.ControlNameSavepoint: - if new.Savepoint != nil { - if new.Savepoint.State == v1beta1.SavepointStateSucceeded { - controlStatus.State = v1beta1.ControlStateSucceeded - setTimestamp(&controlStatus.UpdateTime) - } else if new.Savepoint.State == v1beta1.SavepointStateFailed || new.Savepoint.State == v1beta1.SavepointStateTriggerFailed { - controlStatus.State = v1beta1.ControlStateFailed - setTimestamp(&controlStatus.UpdateTime) - } + if newSavepoint.State == v1beta1.SavepointStateSucceeded { + c.State = v1beta1.ControlStateSucceeded + } else if newSavepoint.TriggerReason == v1beta1.SavepointTriggerReasonUserRequested && + (newSavepoint.State == v1beta1.SavepointStateFailed || newSavepoint.State == v1beta1.SavepointStateTriggerFailed) { + c.State = v1beta1.ControlStateFailed } } - // Aborted by max retry reach - var retries = controlStatus.Details[ControlRetries] - if retries == ControlMaxRetries { - controlStatus.Message = fmt.Sprintf("Aborted control %v. The maximum number of retries has been reached.", controlStatus.Name) - controlStatus.State = v1beta1.ControlStateFailed - setTimestamp(&controlStatus.UpdateTime) + // Update time when state changed. + if c.State != v1beta1.ControlStateInProgress { + setTimestamp(&c.UpdateTime) } - return controlStatus + return c } - // Maintain control status if there is no change. - if recorded.Control != nil && controlStatus == nil { - controlStatus = recorded.Control.DeepCopy() - return controlStatus + if recordedControl != nil && c == nil { + c = recordedControl.DeepCopy() + return c } return nil } +func deriveRevisionStatus( + updateState UpdateState, + observedRevision *Revision, + recordedRevision *v1beta1.RevisionStatus, +) v1beta1.RevisionStatus { + // Derived revision status + var r = v1beta1.RevisionStatus{} + + // Finalize update process. + if updateState == UpdateStateFinished { + r.CurrentRevision = recordedRevision.NextRevision + } + + // Update revision status. + r.NextRevision = getRevisionWithNameNumber(observedRevision.nextRevision) + if r.CurrentRevision == "" { + if recordedRevision.CurrentRevision == "" { + r.CurrentRevision = getRevisionWithNameNumber(observedRevision.currentRevision) + } else { + r.CurrentRevision = recordedRevision.CurrentRevision + } + } + if observedRevision.collisionCount != 0 { + r.CollisionCount = new(int32) + *r.CollisionCount = observedRevision.collisionCount + } + + return r +} + func getStatefulSetState(statefulSet *appsv1.StatefulSet) string { if statefulSet.Status.ReadyReplicas >= *statefulSet.Spec.Replicas { return v1beta1.ComponentStateReady diff --git a/controllers/flinkcluster_util.go b/controllers/flinkcluster_util.go index 5973d5a0..8a9f7098 100644 --- a/controllers/flinkcluster_util.go +++ b/controllers/flinkcluster_util.go @@ -21,8 +21,8 @@ import ( "encoding/json" "fmt" v1beta1 "github.com/googlecloudplatform/flink-operator/api/v1beta1" + "github.com/googlecloudplatform/flink-operator/controllers/flinkclient" "github.com/googlecloudplatform/flink-operator/controllers/history" - "gopkg.in/yaml.v2" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -42,16 +42,21 @@ const ( RevisionNameLabel = "flinkoperator.k8s.io/revision-name" - SavepointMaxAgeForUpdateSecondsDefault = 300 // 5 min - SavepointRequestRetryIntervalSec = 10 + SavepointRequestRetryIntervalSec = 10 ) type UpdateState string +type JobSubmitState string const ( UpdateStatePreparing UpdateState = "Preparing" UpdateStateInProgress UpdateState = "InProgress" UpdateStateFinished UpdateState = "Finished" + + JobDeployStateInProgress = "InProgress" + JobDeployStateSucceeded = "Succeeded" + JobDeployStateFailed = "Failed" + JobDeployStateUnknown = "Unknown" ) type objectForPatch struct { @@ -132,6 +137,10 @@ func setTimestamp(target *string) { *target = tc.ToString(now) } +func isBlank(s *string) bool { + return s == nil || *s == "" || strings.TrimSpace(*s) == "" +} + // Checks whether it is possible to take savepoint. func canTakeSavepoint(cluster v1beta1.FlinkCluster) bool { var jobSpec = cluster.Spec.Job @@ -142,30 +151,23 @@ func canTakeSavepoint(cluster v1beta1.FlinkCluster) bool { (savepointStatus == nil || savepointStatus.State != v1beta1.SavepointStateInProgress) } +func shouldStopJob(cluster *v1beta1.FlinkCluster) bool { + var userControl = cluster.Annotations[v1beta1.ControlAnnotation] + var cancelRequested = cluster.Spec.Job.CancelRequested + return userControl == v1beta1.ControlNameJobCancel || + (cancelRequested != nil && *cancelRequested) +} + // shouldRestartJob returns true if the controller should restart failed or lost job. -// The controller can restart the job only if there is a savepoint to restore, recorded in status field. +// The controller can restart the job only if there is a fresh savepoint to restore, recorded in status field. func shouldRestartJob( - restartPolicy *v1beta1.JobRestartPolicy, + jobSpec *v1beta1.JobSpec, jobStatus *v1beta1.JobStatus) bool { - return restartPolicy != nil && - *restartPolicy == v1beta1.JobRestartPolicyFromSavepointOnFailure && - jobStatus != nil && - (jobStatus.State == v1beta1.JobStateFailed || jobStatus.State == v1beta1.JobStateLost) && - len(jobStatus.SavepointLocation) > 0 -} - -// shouldUpdateJob returns true if the controller should update the job. -// The controller should update the job when update is triggered and it is prepared to update. -// When the job is stopped, no savepoint is required, or the savepoint recorded in status field is up to date, it is ready to update. -func shouldUpdateJob(observed ObservedClusterState) bool { - var jobStatus = observed.cluster.Status.Components.Job - var jobSpec = observed.cluster.Spec.Job - var takeSavepointOnUpdate = jobSpec.TakeSavepointOnUpdate == nil || *jobSpec.TakeSavepointOnUpdate == true - var readyToUpdate = jobStatus == nil || - isJobStopped(jobStatus) || - !takeSavepointOnUpdate || - isSavepointUpToDate(observed.observeTime, jobSpec, jobStatus) - return isUpdateTriggered(observed.cluster.Status) && readyToUpdate + var restartEnabled = jobSpec.RestartPolicy != nil && *jobSpec.RestartPolicy == v1beta1.JobRestartPolicyFromSavepointOnFailure + if restartEnabled && isJobFailed(jobStatus) && isSavepointUpToDate(jobSpec, jobStatus) { + return true + } + return false } func getFromSavepoint(jobSpec batchv1.JobSpec) string { @@ -242,12 +244,12 @@ func getNextRevisionNumber(revisions []*appsv1.ControllerRevision) int64 { return revisions[count-1].Revision + 1 } -func getCurrentRevisionName(status v1beta1.FlinkClusterStatus) string { - return status.CurrentRevision[:strings.LastIndex(status.CurrentRevision, "-")] +func getCurrentRevisionName(r *v1beta1.RevisionStatus) string { + return r.CurrentRevision[:strings.LastIndex(r.CurrentRevision, "-")] } -func getNextRevisionName(status v1beta1.FlinkClusterStatus) string { - return status.NextRevision[:strings.LastIndex(status.NextRevision, "-")] +func getNextRevisionName(r *v1beta1.RevisionStatus) string { + return r.NextRevision[:strings.LastIndex(r.NextRevision, "-")] } // Compose revision in FlinkClusterStatus with name and number of ControllerRevision @@ -270,7 +272,8 @@ func getRetryCount(data map[string]string) (string, error) { return retries, err } -func getNewUserControlRequest(cluster *v1beta1.FlinkCluster) string { +// getNewControlRequest returns new requested control that is not in progress now. +func getNewControlRequest(cluster *v1beta1.FlinkCluster) string { var userControl = cluster.Annotations[v1beta1.ControlAnnotation] var recorded = cluster.Status if recorded.Control == nil || recorded.Control.State != v1beta1.ControlStateInProgress { @@ -279,7 +282,7 @@ func getNewUserControlRequest(cluster *v1beta1.FlinkCluster) string { return "" } -func getUserControlStatus(controlName string, state string) *v1beta1.FlinkClusterControlStatus { +func getControlStatus(controlName string, state string) *v1beta1.FlinkClusterControlStatus { var controlStatus = new(v1beta1.FlinkClusterControlStatus) controlStatus.Name = controlName controlStatus.State = state @@ -287,27 +290,6 @@ func getUserControlStatus(controlName string, state string) *v1beta1.FlinkCluste return controlStatus } -func getNewSavepointStatus(jobID string, triggerID string, triggerReason string, message string, triggerSuccess bool) *v1beta1.SavepointStatus { - var savepointState string - var now string - setTimestamp(&now) - if triggerSuccess { - savepointState = v1beta1.SavepointStateInProgress - } else { - savepointState = v1beta1.SavepointStateTriggerFailed - } - var savepointStatus = &v1beta1.SavepointStatus{ - JobID: jobID, - TriggerID: triggerID, - TriggerReason: triggerReason, - TriggerTime: now, - UpdateTime: now, - Message: message, - State: savepointState, - } - return savepointStatus -} - func getControlEvent(status v1beta1.FlinkClusterControlStatus) (eventType string, eventReason string, eventMessage string) { var msg = status.Message if len(msg) > 100 { @@ -364,33 +346,36 @@ func getSavepointEvent(status v1beta1.SavepointStatus) (eventType string, eventR return } -func isJobActive(status *v1beta1.JobStatus) bool { - return status != nil && - (status.State == v1beta1.JobStateRunning || status.State == v1beta1.JobStatePending) +func isJobActive(j *v1beta1.JobStatus) bool { + return j != nil && + (j.State == v1beta1.JobStateRunning || j.State == v1beta1.JobStateDeploying) } -func isJobStopped(status *v1beta1.JobStatus) bool { - return status != nil && - (status.State == v1beta1.JobStateSucceeded || - status.State == v1beta1.JobStateFailed || - status.State == v1beta1.JobStateCancelled || - status.State == v1beta1.JobStateSuspended || - status.State == v1beta1.JobStateLost) +func isJobPending(j *v1beta1.JobStatus) bool { + return j != nil && + (j.State == v1beta1.JobStatePending || + j.State == v1beta1.JobStateUpdating || + j.State == v1beta1.JobStateRestarting) } -func isJobCancelRequested(cluster v1beta1.FlinkCluster) bool { - var userControl = cluster.Annotations[v1beta1.ControlAnnotation] - var cancelRequested = cluster.Spec.Job.CancelRequested - return userControl == v1beta1.ControlNameJobCancel || - (cancelRequested != nil && *cancelRequested) +func isJobFailed(j *v1beta1.JobStatus) bool { + return j != nil && + (j.State == v1beta1.JobStateFailed || + j.State == v1beta1.JobStateLost || + j.State == v1beta1.JobStateDeployFailed) } -func isJobTerminated(restartPolicy *v1beta1.JobRestartPolicy, jobStatus *v1beta1.JobStatus) bool { - return isJobStopped(jobStatus) && !shouldRestartJob(restartPolicy, jobStatus) +func isJobStopped(j *v1beta1.JobStatus) bool { + return j != nil && + (j.State == v1beta1.JobStateSucceeded || + j.State == v1beta1.JobStateCancelled || + j.State == v1beta1.JobStateFailed || + j.State == v1beta1.JobStateLost || + j.State == v1beta1.JobStateDeployFailed) } -func isUpdateTriggered(status v1beta1.FlinkClusterStatus) bool { - return status.CurrentRevision != status.NextRevision +func isUpdateTriggered(r *v1beta1.RevisionStatus) bool { + return r.CurrentRevision != r.NextRevision } func isUserControlFinished(controlStatus *v1beta1.FlinkClusterControlStatus) bool { @@ -399,17 +384,21 @@ func isUserControlFinished(controlStatus *v1beta1.FlinkClusterControlStatus) boo } // Check if the savepoint is up to date. -func isSavepointUpToDate(now time.Time, jobSpec *v1beta1.JobSpec, jobStatus *v1beta1.JobStatus) bool { - if jobStatus.SavepointLocation != "" && jobStatus.LastSavepointTime != "" { - var spMaxAge int - if jobSpec.SavepointMaxAgeForUpdateSeconds != nil { - spMaxAge = int(*jobSpec.SavepointMaxAgeForUpdateSeconds) - } else { - spMaxAge = SavepointMaxAgeForUpdateSecondsDefault - } - if !hasTimeElapsed(jobStatus.LastSavepointTime, now, spMaxAge) { - return true - } +func isSavepointUpToDate(jobSpec *v1beta1.JobSpec, jobStatus *v1beta1.JobStatus) bool { + if jobStatus.FinalSavepoint { + return true + } + if jobSpec.MaxStateAgeToRestoreSeconds == nil || + jobStatus.SavepointLocation == "" || + jobStatus.SavepointTime == "" || + jobStatus.EndTime == "" { + return false + } + var tc = &TimeConverter{} + var jobEndTime = tc.FromString(jobStatus.EndTime) + var stateMaxAge = int(*jobSpec.MaxStateAgeToRestoreSeconds) + if !hasTimeElapsed(jobStatus.SavepointTime, jobEndTime, stateMaxAge) { + return true } return false } @@ -429,8 +418,8 @@ func hasTimeElapsed(timeToCheckStr string, now time.Time, intervalSec int) bool // If the component is observed as well as the next revision name in status.nextRevision and component's label `flinkoperator.k8s.io/hash` are equal, then it is updated already. // If the component is not observed and it is required, then it is not updated yet. // If the component is not observed and it is optional, but it is specified in the spec, then it is not updated yet. -func isComponentUpdated(component runtime.Object, cluster v1beta1.FlinkCluster) bool { - if !isUpdateTriggered(cluster.Status) { +func isComponentUpdated(component runtime.Object, cluster *v1beta1.FlinkCluster) bool { + if !isUpdateTriggered(&cluster.Status.Revision) { return true } switch o := component.(type) { @@ -467,14 +456,14 @@ func isComponentUpdated(component runtime.Object, cluster v1beta1.FlinkCluster) } var labels, err = meta.NewAccessor().Labels(component) - var nextRevisionName = getNextRevisionName(cluster.Status) + var nextRevisionName = getNextRevisionName(&cluster.Status.Revision) if err != nil { return false } return labels[RevisionNameLabel] == nextRevisionName } -func areComponentsUpdated(components []runtime.Object, cluster v1beta1.FlinkCluster) bool { +func areComponentsUpdated(components []runtime.Object, cluster *v1beta1.FlinkCluster) bool { for _, c := range components { if !isComponentUpdated(c, cluster) { return false @@ -490,14 +479,14 @@ func isUpdatedAll(observed ObservedClusterState) bool { observed.tmStatefulSet, observed.jmService, observed.jmIngress, - observed.job, + observed.flinkJobSubmitter.job, } - return areComponentsUpdated(components, *observed.cluster) + return areComponentsUpdated(components, observed.cluster) } // isClusterUpdateToDate checks whether all cluster components are replaced to next revision. -func isClusterUpdateToDate(observed ObservedClusterState) bool { - if !isUpdateTriggered(observed.cluster.Status) { +func isClusterUpdateToDate(observed *ObservedClusterState) bool { + if !isUpdateTriggered(&observed.cluster.Status.Revision) { return true } components := []runtime.Object{ @@ -506,30 +495,55 @@ func isClusterUpdateToDate(observed ObservedClusterState) bool { observed.tmStatefulSet, observed.jmService, } - return areComponentsUpdated(components, *observed.cluster) + return areComponentsUpdated(components, observed.cluster) } // isFlinkAPIReady checks whether cluster is ready to submit job. -func isFlinkAPIReady(observed ObservedClusterState) bool { +func isFlinkAPIReady(list *flinkclient.JobStatusList) bool { // If the observed Flink job status list is not nil (e.g., emtpy list), // it means Flink REST API server is up and running. It is the source of // truth of whether we can submit a job. - return observed.flinkJobStatus.flinkJobList != nil + return list != nil } -func getUpdateState(observed ObservedClusterState) UpdateState { - var recordedJobStatus = observed.cluster.Status.Components.Job +// jobStateFinalized returns true, if job state is saved so that it can be resumed later. +func finalSavepointRequested(jobID string, s *v1beta1.SavepointStatus) bool { + return s != nil && s.JobID == jobID && + (s.TriggerReason == v1beta1.SavepointTriggerReasonUpdate || + s.TriggerReason == v1beta1.SavepointTriggerReasonJobCancel) +} - switch { - case !isUpdateTriggered(observed.cluster.Status): +func updateReady(jobSpec *v1beta1.JobSpec, job *v1beta1.JobStatus) bool { + var takeSavepoint bool + if jobSpec != nil { + takeSavepoint = jobSpec.TakeSavepointOnUpdate == nil || *jobSpec.TakeSavepointOnUpdate + } + return job == nil || + !isBlank(jobSpec.FromSavepoint) || + !takeSavepoint || + (isJobActive(job) && job.FinalSavepoint) || + (!isJobActive(job) && isSavepointUpToDate(jobSpec, job)) +} + +func getUpdateState(observed *ObservedClusterState) UpdateState { + if observed.cluster == nil { + return "" + } + var recorded = observed.cluster.Status + var revision = recorded.Revision + var job = recorded.Components.Job + var jobSpec = observed.cluster.Spec.Job + + if !isUpdateTriggered(&revision) { return "" - case isJobActive(recordedJobStatus): + } + switch { + case isJobActive(job) || !updateReady(jobSpec, job): return UpdateStatePreparing - case isClusterUpdateToDate(observed): - return UpdateStateFinished - default: + case !isClusterUpdateToDate(observed): return UpdateStateInProgress } + return UpdateStateFinished } func getNonLiveHistory(revisions []*appsv1.ControllerRevision, historyLimit int) []*appsv1.ControllerRevision { @@ -548,7 +562,7 @@ func getNonLiveHistory(revisions []*appsv1.ControllerRevision, historyLimit int) func getFlinkJobDeploymentState(flinkJobState string) string { switch flinkJobState { - case "INITIALIZING", "CREATED", "RUNNING", "FAILING", "CANCELLING", "RESTARTING", "RECONCILING": + case "INITIALIZING", "CREATED", "RUNNING", "FAILING", "CANCELLING", "RESTARTING", "RECONCILING", "SUSPENDED": return v1beta1.JobStateRunning case "FINISHED": return v1beta1.JobStateSucceeded @@ -556,36 +570,7 @@ func getFlinkJobDeploymentState(flinkJobState string) string { return v1beta1.JobStateCancelled case "FAILED": return v1beta1.JobStateFailed - case "SUSPENDED": - return v1beta1.JobStateSuspended default: return "" } } - -// getFlinkJobSubmitLog extract submit result from the pod termination log. -func getFlinkJobSubmitLog(observedPod *corev1.Pod) (*FlinkJobSubmitLog, error) { - if observedPod == nil { - return nil, fmt.Errorf("no job pod found, even though submission completed") - } - var containerStatuses = observedPod.Status.ContainerStatuses - if len(containerStatuses) == 0 || - containerStatuses[0].State.Terminated == nil || - containerStatuses[0].State.Terminated.Message == "" { - return nil, fmt.Errorf("job pod found, but no termination log found even though submission completed") - } - - // The job submission script writes the submission log to the pod termination log at the end of execution. - // If the job submission is successful, the extracted job ID is also included. - // The job submit script writes the submission result in YAML format, - // so parse it here to get the ID - if available - and log. - // Note: https://kubernetes.io/docs/tasks/debug-application-cluster/determine-reason-pod-failure/ - var rawJobSubmitResult = containerStatuses[0].State.Terminated.Message - var result = new(FlinkJobSubmitLog) - var err = yaml.Unmarshal([]byte(rawJobSubmitResult), result) - if err != nil { - return nil, err - } - - return result, nil -} diff --git a/controllers/flinkcluster_util_test.go b/controllers/flinkcluster_util_test.go index 47a6faf0..d0f9e6cf 100644 --- a/controllers/flinkcluster_util_test.go +++ b/controllers/flinkcluster_util_test.go @@ -17,7 +17,6 @@ limitations under the License. package controllers import ( - "github.com/googlecloudplatform/flink-operator/controllers/flinkclient" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -48,27 +47,68 @@ func TestTimeConverter(t *testing.T) { } func TestShouldRestartJob(t *testing.T) { + var tc = &TimeConverter{} var restartOnFailure = v1beta1.JobRestartPolicyFromSavepointOnFailure - var jobStatus1 = v1beta1.JobStatus{ + var neverRestart = v1beta1.JobRestartPolicyNever + var maxStateAgeToRestoreSeconds = int32(300) // 5 min + + // Restart with savepoint up to date + var savepointTime = tc.ToString(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + var endTime = tc.ToString(time.Date(2020, 1, 1, 0, 1, 0, 0, time.UTC)) + var jobSpec = v1beta1.JobSpec{ + RestartPolicy: &restartOnFailure, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + var jobStatus = v1beta1.JobStatus{ State: v1beta1.JobStateFailed, SavepointLocation: "gs://my-bucket/savepoint-123", + SavepointTime: savepointTime, + EndTime: endTime, + } + var restart = shouldRestartJob(&jobSpec, &jobStatus) + assert.Equal(t, restart, true) + + // Not restart without savepoint + jobSpec = v1beta1.JobSpec{ + RestartPolicy: &restartOnFailure, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, } - var restart1 = shouldRestartJob(&restartOnFailure, &jobStatus1) - assert.Equal(t, restart1, true) + jobStatus = v1beta1.JobStatus{ + State: v1beta1.JobStateFailed, + EndTime: endTime, + } + restart = shouldRestartJob(&jobSpec, &jobStatus) + assert.Equal(t, restart, false) - var jobStatus2 = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, + // Not restart with restartPolicy Never + jobSpec = v1beta1.JobSpec{ + RestartPolicy: &neverRestart, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + jobStatus = v1beta1.JobStatus{ + State: v1beta1.JobStateFailed, + SavepointLocation: "gs://my-bucket/savepoint-123", + SavepointTime: savepointTime, + EndTime: endTime, } - var restart2 = shouldRestartJob(&restartOnFailure, &jobStatus2) - assert.Equal(t, restart2, false) + restart = shouldRestartJob(&jobSpec, &jobStatus) + assert.Equal(t, restart, false) - var neverRestart = v1beta1.JobRestartPolicyNever - var jobStatus3 = v1beta1.JobStatus{ + // Not restart with old savepoint + savepointTime = tc.ToString(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) + endTime = tc.ToString(time.Date(2020, 1, 1, 0, 5, 0, 0, time.UTC)) + jobSpec = v1beta1.JobSpec{ + RestartPolicy: &neverRestart, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + jobStatus = v1beta1.JobStatus{ State: v1beta1.JobStateFailed, SavepointLocation: "gs://my-bucket/savepoint-123", + SavepointTime: savepointTime, + EndTime: endTime, } - var restart3 = shouldRestartJob(&neverRestart, &jobStatus3) - assert.Equal(t, restart3, false) + restart = shouldRestartJob(&jobSpec, &jobStatus) + assert.Equal(t, restart, false) } func TestGetRetryCount(t *testing.T) { @@ -220,108 +260,6 @@ func TestCanTakeSavepoint(t *testing.T) { assert.Equal(t, take, false) } -func TestShouldUpdateJob(t *testing.T) { - // should update - var tc = &TimeConverter{} - var savepointTime = time.Now() - var observeTime = savepointTime.Add(time.Second * 100) - var savepointMaxAgeForUpdateSeconds = int32(300) - var observed = ObservedClusterState{ - observeTime: observeTime, - cluster: &v1beta1.FlinkCluster{ - Spec: v1beta1.FlinkClusterSpec{Job: &v1beta1.JobSpec{ - SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, - }}, - Status: v1beta1.FlinkClusterStatus{ - Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{ - State: v1beta1.JobStateRunning, - LastSavepointTime: tc.ToString(savepointTime), - SavepointLocation: "gs://my-bucket/savepoint-123", - }}, - CurrentRevision: "1", NextRevision: "2", - }, - }, - } - var update = shouldUpdateJob(observed) - assert.Equal(t, update, true) - - // should update when update triggered and job failed. - observed = ObservedClusterState{ - cluster: &v1beta1.FlinkCluster{ - Spec: v1beta1.FlinkClusterSpec{Job: &v1beta1.JobSpec{ - SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, - }}, - Status: v1beta1.FlinkClusterStatus{ - Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - }}, - CurrentRevision: "1", NextRevision: "2", - }, - }, - } - update = shouldUpdateJob(observed) - assert.Equal(t, update, true) - - // cannot update with old savepoint - tc = &TimeConverter{} - savepointTime = time.Now() - observeTime = savepointTime.Add(time.Second * 500) - observed = ObservedClusterState{ - observeTime: observeTime, - cluster: &v1beta1.FlinkCluster{ - Spec: v1beta1.FlinkClusterSpec{Job: &v1beta1.JobSpec{ - SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, - }}, - Status: v1beta1.FlinkClusterStatus{ - Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{ - State: v1beta1.JobStateRunning, - LastSavepointTime: tc.ToString(savepointTime), - SavepointLocation: "gs://my-bucket/savepoint-123", - }}, - CurrentRevision: "1", NextRevision: "2", - }, - }, - } - update = shouldUpdateJob(observed) - assert.Equal(t, update, false) - - // cannot update without savepointLocation - observed = ObservedClusterState{ - cluster: &v1beta1.FlinkCluster{ - Spec: v1beta1.FlinkClusterSpec{Job: &v1beta1.JobSpec{ - SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, - }}, - Status: v1beta1.FlinkClusterStatus{ - Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{ - State: v1beta1.JobStateUpdating, - }}, - CurrentRevision: "1", NextRevision: "2", - }, - }, - } - update = shouldUpdateJob(observed) - assert.Equal(t, update, false) - - // proceed update without savepointLocation if takeSavepoint is false. - takeSavepointOnUpdate := false - observed = ObservedClusterState{ - cluster: &v1beta1.FlinkCluster{ - Spec: v1beta1.FlinkClusterSpec{Job: &v1beta1.JobSpec{ - TakeSavepointOnUpdate: &takeSavepointOnUpdate, - SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, - }}, - Status: v1beta1.FlinkClusterStatus{ - Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{ - State: v1beta1.JobStateUpdating, - }}, - CurrentRevision: "1", NextRevision: "2", - }, - }, - } - update = shouldUpdateJob(observed) - assert.Equal(t, update, true) -} - func TestGetNextRevisionNumber(t *testing.T) { var revisions []*appsv1.ControllerRevision var nextRevision = getNextRevisionNumber(revisions) @@ -332,173 +270,90 @@ func TestGetNextRevisionNumber(t *testing.T) { assert.Equal(t, nextRevision, int64(3)) } -func TestIsJobTerminated(t *testing.T) { - var jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateSucceeded, - } - var terminated = isJobTerminated(nil, &jobStatus) - assert.Equal(t, terminated, true) - - var restartOnFailure = v1beta1.JobRestartPolicyFromSavepointOnFailure - jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - SavepointLocation: "gs://my-bucket/savepoint-123", - } - terminated = isJobTerminated(&restartOnFailure, &jobStatus) - assert.Equal(t, terminated, false) -} - func TestIsSavepointUpToDate(t *testing.T) { var tc = &TimeConverter{} var savepointTime = time.Now() - var observeTime = savepointTime.Add(time.Second * 100) - var savepointMaxAgeForUpdateSeconds = int32(300) + var jobEndTime = savepointTime.Add(time.Second * 100) + var maxStateAgeToRestoreSeconds = int32(300) var jobSpec = v1beta1.JobSpec{ - SavepointMaxAgeForUpdateSeconds: &savepointMaxAgeForUpdateSeconds, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, } var jobStatus = v1beta1.JobStatus{ State: v1beta1.JobStateFailed, - LastSavepointTime: tc.ToString(savepointTime), + SavepointTime: tc.ToString(savepointTime), SavepointLocation: "gs://my-bucket/savepoint-123", } - var update = isSavepointUpToDate(observeTime, &jobSpec, &jobStatus) + var update = isSavepointUpToDate(&jobSpec, &jobStatus) assert.Equal(t, update, true) // old savepointTime = time.Now() - observeTime = savepointTime.Add(time.Second * 500) + jobEndTime = savepointTime.Add(time.Second * 500) jobStatus = v1beta1.JobStatus{ State: v1beta1.JobStateFailed, - LastSavepointTime: tc.ToString(savepointTime), + SavepointTime: tc.ToString(savepointTime), SavepointLocation: "gs://my-bucket/savepoint-123", + EndTime: tc.ToString(jobEndTime), } - update = isSavepointUpToDate(observeTime, &jobSpec, &jobStatus) + update = isSavepointUpToDate(&jobSpec, &jobStatus) assert.Equal(t, update, false) // Fails without savepointLocation savepointTime = time.Now() - observeTime = savepointTime.Add(time.Second * 500) + jobEndTime = savepointTime.Add(time.Second * 500) jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - LastSavepointTime: tc.ToString(savepointTime), + State: v1beta1.JobStateFailed, + SavepointTime: tc.ToString(savepointTime), + EndTime: tc.ToString(jobEndTime), } - update = isSavepointUpToDate(observeTime, &jobSpec, &jobStatus) + update = isSavepointUpToDate(&jobSpec, &jobStatus) assert.Equal(t, update, false) } func TestIsComponentUpdated(t *testing.T) { var cluster = v1beta1.FlinkCluster{ - Status: v1beta1.FlinkClusterStatus{NextRevision: "cluster-85dc8f749-2"}, + Status: v1beta1.FlinkClusterStatus{Revision: v1beta1.RevisionStatus{NextRevision: "cluster-85dc8f749-2"}}, } var cluster2 = v1beta1.FlinkCluster{ Spec: v1beta1.FlinkClusterSpec{ JobManager: v1beta1.JobManagerSpec{Ingress: &v1beta1.JobManagerIngressSpec{}}, Job: &v1beta1.JobSpec{}, }, - Status: v1beta1.FlinkClusterStatus{NextRevision: "cluster-85dc8f749-2"}, + Status: v1beta1.FlinkClusterStatus{Revision: v1beta1.RevisionStatus{NextRevision: "cluster-85dc8f749-2"}}, } var deploy = &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{ RevisionNameLabel: "cluster-85dc8f749", }}} - var update = isComponentUpdated(deploy, cluster) + var update = isComponentUpdated(deploy, &cluster) assert.Equal(t, update, true) deploy = &appsv1.Deployment{} - update = isComponentUpdated(deploy, cluster) + update = isComponentUpdated(deploy, &cluster) assert.Equal(t, update, false) deploy = nil - update = isComponentUpdated(deploy, cluster) + update = isComponentUpdated(deploy, &cluster) assert.Equal(t, update, false) var job = &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{ RevisionNameLabel: "cluster-85dc8f749", }}} - update = isComponentUpdated(job, cluster2) + update = isComponentUpdated(job, &cluster2) assert.Equal(t, update, true) job = &batchv1.Job{} - update = isComponentUpdated(job, cluster2) + update = isComponentUpdated(job, &cluster2) assert.Equal(t, update, false) job = nil - update = isComponentUpdated(job, cluster2) + update = isComponentUpdated(job, &cluster2) assert.Equal(t, update, false) job = nil - update = isComponentUpdated(job, cluster) + update = isComponentUpdated(job, &cluster) assert.Equal(t, update, true) } -func TestIsFlinkAPIReady(t *testing.T) { - var observed = ObservedClusterState{ - cluster: &v1beta1.FlinkCluster{ - Spec: v1beta1.FlinkClusterSpec{ - JobManager: v1beta1.JobManagerSpec{Ingress: &v1beta1.JobManagerIngressSpec{}}, - Job: &v1beta1.JobSpec{}, - }, - Status: v1beta1.FlinkClusterStatus{NextRevision: "cluster-85dc8f749-2"}, - }, - configMap: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - jmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - tmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - jmService: &corev1.Service{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - flinkJobStatus: FlinkJobStatus{flinkJobList: &flinkclient.JobStatusList{}}, - } - var ready = isFlinkAPIReady(observed) - assert.Equal(t, ready, true) - - // flinkJobList is nil - observed = ObservedClusterState{ - cluster: &v1beta1.FlinkCluster{ - Spec: v1beta1.FlinkClusterSpec{ - JobManager: v1beta1.JobManagerSpec{Ingress: &v1beta1.JobManagerIngressSpec{}}, - Job: &v1beta1.JobSpec{}, - }, - Status: v1beta1.FlinkClusterStatus{NextRevision: "cluster-85dc8f749-2"}, - }, - configMap: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - jmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - tmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - jmService: &corev1.Service{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - } - ready = isFlinkAPIReady(observed) - assert.Equal(t, ready, false) - - // jmStatefulSet is not observed - observed = ObservedClusterState{ - cluster: &v1beta1.FlinkCluster{ - Spec: v1beta1.FlinkClusterSpec{ - JobManager: v1beta1.JobManagerSpec{Ingress: &v1beta1.JobManagerIngressSpec{}}, - Job: &v1beta1.JobSpec{}, - }, - Status: v1beta1.FlinkClusterStatus{NextRevision: "cluster-85dc8f749-2"}, - }, - configMap: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - tmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - jmService: &corev1.Service{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - } - ready = isFlinkAPIReady(observed) - assert.Equal(t, ready, false) - - // jmStatefulSet is not updated - observed = ObservedClusterState{ - cluster: &v1beta1.FlinkCluster{ - Spec: v1beta1.FlinkClusterSpec{ - JobManager: v1beta1.JobManagerSpec{Ingress: &v1beta1.JobManagerIngressSpec{}}, - Job: &v1beta1.JobSpec{}, - }, - Status: v1beta1.FlinkClusterStatus{NextRevision: "cluster-85dc8f749-2"}, - }, - configMap: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - jmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, - tmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - jmService: &corev1.Service{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - } - ready = isFlinkAPIReady(observed) - assert.Equal(t, ready, false) -} - func TestGetUpdateState(t *testing.T) { var observed = ObservedClusterState{ cluster: &v1beta1.FlinkCluster{ @@ -507,16 +362,16 @@ func TestGetUpdateState(t *testing.T) { Job: &v1beta1.JobSpec{}, }, Status: v1beta1.FlinkClusterStatus{ - Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{State: v1beta1.JobStateRunning}}, - CurrentRevision: "cluster-85dc8f749-2", NextRevision: "cluster-aa5e3a87z-3"}, + Components: v1beta1.FlinkClusterComponentsStatus{Job: &v1beta1.JobStatus{State: v1beta1.JobStateRunning}}, + Revision: v1beta1.RevisionStatus{CurrentRevision: "cluster-85dc8f749-2", NextRevision: "cluster-aa5e3a87z-3"}}, }, - job: &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - configMap: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - jmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - tmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, - jmService: &corev1.Service{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, + flinkJobSubmitter: FlinkJobSubmitter{job: &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}}, + configMap: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, + jmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, + tmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, + jmService: &corev1.Service{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, } - var state = getUpdateState(observed) + var state = getUpdateState(&observed) assert.Equal(t, state, UpdateStatePreparing) observed = ObservedClusterState{ @@ -525,13 +380,13 @@ func TestGetUpdateState(t *testing.T) { JobManager: v1beta1.JobManagerSpec{Ingress: &v1beta1.JobManagerIngressSpec{}}, Job: &v1beta1.JobSpec{}, }, - Status: v1beta1.FlinkClusterStatus{CurrentRevision: "cluster-85dc8f749-2", NextRevision: "cluster-aa5e3a87z-3"}, + Status: v1beta1.FlinkClusterStatus{Revision: v1beta1.RevisionStatus{CurrentRevision: "cluster-85dc8f749-2", NextRevision: "cluster-aa5e3a87z-3"}}, }, jmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, tmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, jmService: &corev1.Service{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}}, } - state = getUpdateState(observed) + state = getUpdateState(&observed) assert.Equal(t, state, UpdateStateInProgress) observed = ObservedClusterState{ @@ -540,16 +395,16 @@ func TestGetUpdateState(t *testing.T) { JobManager: v1beta1.JobManagerSpec{Ingress: &v1beta1.JobManagerIngressSpec{}}, Job: &v1beta1.JobSpec{}, }, - Status: v1beta1.FlinkClusterStatus{CurrentRevision: "cluster-85dc8f749-2", NextRevision: "cluster-aa5e3a87z-3"}, + Status: v1beta1.FlinkClusterStatus{Revision: v1beta1.RevisionStatus{CurrentRevision: "cluster-85dc8f749-2", NextRevision: "cluster-aa5e3a87z-3"}}, }, - job: &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, - configMap: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, - jmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, - tmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, - jmService: &corev1.Service{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, - jmIngress: &extensionsv1beta1.Ingress{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, - } - state = getUpdateState(observed) + flinkJobSubmitter: FlinkJobSubmitter{job: &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}}, + configMap: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, + jmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, + tmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, + jmService: &corev1.Service{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, + jmIngress: &extensionsv1beta1.Ingress{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, + } + state = getUpdateState(&observed) assert.Equal(t, state, UpdateStateFinished) } @@ -602,54 +457,3 @@ func TestGetNonLiveHistory(t *testing.T) { nonLiveHistory = getNonLiveHistory(revisions, historyLimit) assert.Equal(t, len(nonLiveHistory), 0) } - -func TestGetFlinkJobDeploymentState(t *testing.T) { - var pod corev1.Pod - var submit, expected *FlinkJobSubmitLog - var err error - var termMsg string - - // success - termMsg = ` -jobID: ec74209eb4e3db8ae72db00bd7a830aa -message: | - Successfully submitted! - /opt/flink/bin/flink run --jobmanager flinkjobcluster-sample-jobmanager:8081 --class org.apache.flink.streaming.examples.wordcount.WordCount --parallelism 2 --detached ./examples/streaming/WordCount.jar --input ./README.txt - Starting execution of program - Printing result to stdout. Use --output to specify output path. - Job has been submitted with JobID ec74209eb4e3db8ae72db00bd7a830aa -` - expected = &FlinkJobSubmitLog{ - JobID: "ec74209eb4e3db8ae72db00bd7a830aa", - Message: `Successfully submitted! -/opt/flink/bin/flink run --jobmanager flinkjobcluster-sample-jobmanager:8081 --class org.apache.flink.streaming.examples.wordcount.WordCount --parallelism 2 --detached ./examples/streaming/WordCount.jar --input ./README.txt -Starting execution of program -Printing result to stdout. Use --output to specify output path. -Job has been submitted with JobID ec74209eb4e3db8ae72db00bd7a830aa -`, - } - pod = corev1.Pod{ - Status: corev1.PodStatus{ - ContainerStatuses: []corev1.ContainerStatus{{ - State: corev1.ContainerState{ - Terminated: &corev1.ContainerStateTerminated{ - Message: termMsg, - }}}}}} - submit, _ = getFlinkJobSubmitLog(&pod) - assert.DeepEqual(t, *submit, *expected) - - // failed: pod not found - submit, err = getFlinkJobSubmitLog(nil) - assert.Error(t, err, "no job pod found, even though submission completed") - - // failed: message not found - pod = corev1.Pod{ - Status: corev1.PodStatus{ - ContainerStatuses: []corev1.ContainerStatus{{ - State: corev1.ContainerState{ - Terminated: &corev1.ContainerStateTerminated{ - Message: "", - }}}}}} - submit, err = getFlinkJobSubmitLog(&pod) - assert.Error(t, err, "job pod found, but no termination log found even though submission completed") -} From 308b909be4251837f92d8526daa484bd6229a3d1 Mon Sep 17 00:00:00 2001 From: elanv Date: Thu, 18 Mar 2021 10:25:11 +0900 Subject: [PATCH 4/6] fix types --- api/v1beta1/flinkcluster_types.go | 2 +- .../flinkoperator.k8s.io_flinkclusters.yaml | 257 +++++++++++++++--- 2 files changed, 217 insertions(+), 42 deletions(-) diff --git a/api/v1beta1/flinkcluster_types.go b/api/v1beta1/flinkcluster_types.go index 2789c37e..ecd44d03 100644 --- a/api/v1beta1/flinkcluster_types.go +++ b/api/v1beta1/flinkcluster_types.go @@ -681,7 +681,7 @@ type FlinkClusterStatus struct { Savepoint *SavepointStatus `json:"savepoint,omitempty"` // The status of revision. - Revision RevisionStatus `json:"revision"` + Revision RevisionStatus `json:"revision,omitempty"` // Last update timestamp for this status. LastUpdateTime string `json:"lastUpdateTime,omitempty"` diff --git a/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml b/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml index 684709a7..36e67f03 100644 --- a/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml +++ b/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.4 + controller-gen.kubebuilder.io/version: v0.3.0 creationTimestamp: null name: flinkclusters.flinkoperator.k8s.io spec: @@ -85,7 +85,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -217,7 +221,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -453,6 +461,10 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -522,11 +534,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -714,11 +734,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object restartPolicy: @@ -960,7 +988,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -976,7 +1008,11 @@ spec: medium: type: string sizeLimit: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object fc: properties: @@ -1201,7 +1237,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -1470,7 +1510,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -1706,6 +1750,10 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -1775,11 +1823,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -1946,7 +2002,11 @@ spec: type: object type: array memoryOffHeapMin: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true memoryOffHeapRatio: format: int32 type: integer @@ -1984,11 +2044,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -2089,7 +2157,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -2325,6 +2397,10 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -2394,11 +2470,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -2611,11 +2695,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object selector: @@ -2656,7 +2748,11 @@ spec: type: array capacity: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object conditions: items: @@ -2862,7 +2958,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -2878,7 +2978,11 @@ spec: medium: type: string sizeLimit: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object fc: properties: @@ -3103,7 +3207,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -3367,7 +3475,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -3603,6 +3715,10 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -3672,11 +3788,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -3843,7 +3967,11 @@ spec: type: object type: array memoryOffHeapMin: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true memoryOffHeapRatio: format: int32 type: integer @@ -3878,11 +4006,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -3983,7 +4119,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -4219,6 +4359,10 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: properties: exec: @@ -4288,11 +4432,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object securityContext: @@ -4505,11 +4657,19 @@ spec: properties: limits: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object requests: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: object selector: @@ -4550,7 +4710,11 @@ spec: type: array capacity: additionalProperties: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object conditions: items: @@ -4756,7 +4920,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -4772,7 +4940,11 @@ spec: medium: type: string sizeLimit: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object fc: properties: @@ -4997,7 +5169,11 @@ spec: containerName: type: string divisor: - type: string + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true resource: type: string required: @@ -5340,7 +5516,6 @@ spec: type: string required: - components - - revision - state type: object required: From 58eec1b1ee41530af5863e2d71b89b39a828e1bc Mon Sep 17 00:00:00 2001 From: elanv Date: Sun, 28 Mar 2021 00:16:52 +0900 Subject: [PATCH 5/6] unit test --- controllers/flinkcluster_observer_test.go | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 controllers/flinkcluster_observer_test.go diff --git a/controllers/flinkcluster_observer_test.go b/controllers/flinkcluster_observer_test.go new file mode 100644 index 00000000..c1b85335 --- /dev/null +++ b/controllers/flinkcluster_observer_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2019 Google LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "gotest.tools/assert" + corev1 "k8s.io/api/core/v1" + "testing" +) + +func TestGetFlinkJobDeploymentState(t *testing.T) { + var pod corev1.Pod + var submitterLog, expected *SubmitterLog + var err error + var termMsg string + + // success + termMsg = ` +jobID: ec74209eb4e3db8ae72db00bd7a830aa +message: | + Successfully submitted! + /opt/flink/bin/flink run --jobmanager flinkjobcluster-sample-jobmanager:8081 --class org.apache.flink.streaming.examples.wordcount.WordCount --parallelism 2 --detached ./examples/streaming/WordCount.jar --input ./README.txt + Starting execution of program + Printing result to stdout. Use --output to specify output path. + Job has been submitted with JobID ec74209eb4e3db8ae72db00bd7a830aa +` + expected = &SubmitterLog{ + JobID: "ec74209eb4e3db8ae72db00bd7a830aa", + Message: `Successfully submitted! +/opt/flink/bin/flink run --jobmanager flinkjobcluster-sample-jobmanager:8081 --class org.apache.flink.streaming.examples.wordcount.WordCount --parallelism 2 --detached ./examples/streaming/WordCount.jar --input ./README.txt +Starting execution of program +Printing result to stdout. Use --output to specify output path. +Job has been submitted with JobID ec74209eb4e3db8ae72db00bd7a830aa +`, + } + pod = corev1.Pod{ + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{{ + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + Message: termMsg, + }}}}}} + submitterLog = new(SubmitterLog) + _ = observeFlinkJobSubmitterLog(&pod, submitterLog) + assert.DeepEqual(t, *submitterLog, *expected) + + // failed: message not found + pod = corev1.Pod{ + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{{ + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + Message: "", + }}}}}} + err = observeFlinkJobSubmitterLog(&pod, submitterLog) + assert.Error(t, err, "job pod found, but no termination log") +} From f71e84a739753251d3f712879861385fb89852b2 Mon Sep 17 00:00:00 2001 From: elanv Date: Mon, 5 Apr 2021 13:30:53 +0900 Subject: [PATCH 6/6] fix savepoint and update routines - fix handling failed auto savepoint - fix validations and tests related to changes - improve update routine - change the behavior of handling unexpected jobs - add a constraint for update: when takeSavepointOnUpdate is true, latest savepoint age should be less than maxStateAgeToRestore --- api/v1beta1/flinkcluster_default.go | 4 - api/v1beta1/flinkcluster_default_test.go | 33 ++- api/v1beta1/flinkcluster_types.go | 18 +- api/v1beta1/flinkcluster_types_util.go | 147 ++++++++++++ api/v1beta1/flinkcluster_types_util_test.go | 154 ++++++++++++ api/v1beta1/flinkcluster_validate.go | 79 ++++--- api/v1beta1/flinkcluster_validate_test.go | 219 ++++++++++++------ api/v1beta1/zz_generated.deepcopy.go | 15 ++ .../flinkoperator.k8s.io_flinkclusters.yaml | 1 + controllers/flinkcluster_converter.go | 8 +- controllers/flinkcluster_observer.go | 42 ++-- controllers/flinkcluster_observer_test.go | 15 +- controllers/flinkcluster_reconciler.go | 108 +++++---- controllers/flinkcluster_updater.go | 61 +++-- controllers/flinkcluster_util.go | 108 ++------- controllers/flinkcluster_util_test.go | 115 +-------- 16 files changed, 671 insertions(+), 456 deletions(-) create mode 100644 api/v1beta1/flinkcluster_types_util.go create mode 100644 api/v1beta1/flinkcluster_types_util_test.go diff --git a/api/v1beta1/flinkcluster_default.go b/api/v1beta1/flinkcluster_default.go index 744cd0ae..e1d419d9 100644 --- a/api/v1beta1/flinkcluster_default.go +++ b/api/v1beta1/flinkcluster_default.go @@ -128,10 +128,6 @@ func _SetJobDefault(jobSpec *JobSpec) { AfterJobCancelled: CleanupActionDeleteCluster, } } - if jobSpec.MaxStateAgeToRestoreSeconds == nil { - jobSpec.MaxStateAgeToRestoreSeconds = new(int32) - *jobSpec.MaxStateAgeToRestoreSeconds = 300 - } } func _SetHadoopConfigDefault(hadoopConfig *HadoopConfig) { diff --git a/api/v1beta1/flinkcluster_default_test.go b/api/v1beta1/flinkcluster_default_test.go index c4608d13..140866aa 100644 --- a/api/v1beta1/flinkcluster_default_test.go +++ b/api/v1beta1/flinkcluster_default_test.go @@ -53,7 +53,6 @@ func TestSetDefault(t *testing.T) { var defaultJobParallelism = int32(1) var defaultJobNoLoggingToStdout = false var defaultJobRestartPolicy = JobRestartPolicyNever - var defaultMaxStateAgeToRestoreSeconds = int32(300) var defaultMemoryOffHeapRatio = int32(25) var defaultMemoryOffHeapMin = resource.MustParse("600M") var defaultRecreateOnUpdate = true @@ -99,11 +98,10 @@ func TestSetDefault(t *testing.T) { SecurityContext: nil, }, Job: &JobSpec{ - AllowNonRestoredState: &defaultJobAllowNonRestoredState, - Parallelism: &defaultJobParallelism, - NoLoggingToStdout: &defaultJobNoLoggingToStdout, - RestartPolicy: &defaultJobRestartPolicy, - MaxStateAgeToRestoreSeconds: &defaultMaxStateAgeToRestoreSeconds, + AllowNonRestoredState: &defaultJobAllowNonRestoredState, + Parallelism: &defaultJobParallelism, + NoLoggingToStdout: &defaultJobNoLoggingToStdout, + RestartPolicy: &defaultJobRestartPolicy, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: "DeleteCluster", AfterJobFails: "KeepCluster", @@ -143,7 +141,6 @@ func TestSetNonDefault(t *testing.T) { var jobParallelism = int32(2) var jobNoLoggingToStdout = true var jobRestartPolicy = JobRestartPolicyFromSavepointOnFailure - var jobMaxStateAgeToRestoreSeconds = int32(1000) var memoryOffHeapRatio = int32(50) var memoryOffHeapMin = resource.MustParse("600M") var recreateOnUpdate = false @@ -194,12 +191,11 @@ func TestSetNonDefault(t *testing.T) { SecurityContext: &securityContext, }, Job: &JobSpec{ - AllowNonRestoredState: &jobAllowNonRestoredState, - Parallelism: &jobParallelism, - NoLoggingToStdout: &jobNoLoggingToStdout, - RestartPolicy: &jobRestartPolicy, - MaxStateAgeToRestoreSeconds: &jobMaxStateAgeToRestoreSeconds, - SecurityContext: &securityContext, + AllowNonRestoredState: &jobAllowNonRestoredState, + Parallelism: &jobParallelism, + NoLoggingToStdout: &jobNoLoggingToStdout, + RestartPolicy: &jobRestartPolicy, + SecurityContext: &securityContext, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: "DeleteTaskManagers", AfterJobFails: "DeleteCluster", @@ -260,12 +256,11 @@ func TestSetNonDefault(t *testing.T) { SecurityContext: &securityContext, }, Job: &JobSpec{ - AllowNonRestoredState: &jobAllowNonRestoredState, - Parallelism: &jobParallelism, - NoLoggingToStdout: &jobNoLoggingToStdout, - RestartPolicy: &jobRestartPolicy, - MaxStateAgeToRestoreSeconds: &jobMaxStateAgeToRestoreSeconds, - SecurityContext: &securityContext, + AllowNonRestoredState: &jobAllowNonRestoredState, + Parallelism: &jobParallelism, + NoLoggingToStdout: &jobNoLoggingToStdout, + RestartPolicy: &jobRestartPolicy, + SecurityContext: &securityContext, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: "DeleteTaskManagers", AfterJobFails: "DeleteCluster", diff --git a/api/v1beta1/flinkcluster_types.go b/api/v1beta1/flinkcluster_types.go index ecd44d03..b7b33f29 100644 --- a/api/v1beta1/flinkcluster_types.go +++ b/api/v1beta1/flinkcluster_types.go @@ -353,9 +353,14 @@ type JobSpec struct { SavepointsDir *string `json:"savepointsDir,omitempty"` // Should take savepoint before updating job, default: true. + // If this is set as false, maxStateAgeToRestoreSeconds must be provided to limit the savepoint age to restore. TakeSavepointOnUpdate *bool `json:"takeSavepointOnUpdate,omitempty"` - // Maximum age of the savepoint that a job can be restored when the job is restarted or updated from stopped state, default: 300 + // Maximum age of the savepoint that allowed to restore state.. + // This is applied to auto restart on failure, update from stopped state and update without taking savepoint. + // If nil, job can be restarted only when the latest savepoint is the final job state (created by "stop with savepoint") + // - that is, only when job can be resumed from the suspended state. + // +kubebuilder:validation:Minimum=0 MaxStateAgeToRestoreSeconds *int32 `json:"maxStateAgeToRestoreSeconds,omitempty"` // Automatically take a savepoint to the `savepointsDir` every n seconds. @@ -578,7 +583,7 @@ type JobStatus struct { // Last successful savepoint completed timestamp. SavepointTime string `json:"savepointTime,omitempty"` - // The savepoint is the final state of the job. + // The savepoint recorded in savepointLocation is the final state of the job. FinalSavepoint bool `json:"finalSavepoint,omitempty"` // The timestamp of the Flink job deployment that creating job submitter. @@ -711,12 +716,3 @@ type FlinkClusterList struct { func init() { SchemeBuilder.Register(&FlinkCluster{}, &FlinkClusterList{}) } - -func (j *JobStatus) isJobStopped() bool { - return j != nil && - (j.State == JobStateSucceeded || - j.State == JobStateCancelled || - j.State == JobStateFailed || - j.State == JobStateLost || - j.State == JobStateDeployFailed) -} diff --git a/api/v1beta1/flinkcluster_types_util.go b/api/v1beta1/flinkcluster_types_util.go new file mode 100644 index 00000000..880160f0 --- /dev/null +++ b/api/v1beta1/flinkcluster_types_util.go @@ -0,0 +1,147 @@ +package v1beta1 + +import ( + "fmt" + "strings" + "time" +) + +func (j *JobStatus) IsActive() bool { + return j != nil && + (j.State == JobStateRunning || j.State == JobStateDeploying) +} + +func (j *JobStatus) IsPending() bool { + return j != nil && + (j.State == JobStatePending || + j.State == JobStateUpdating || + j.State == JobStateRestarting) +} + +func (j *JobStatus) IsFailed() bool { + return j != nil && + (j.State == JobStateFailed || + j.State == JobStateLost || + j.State == JobStateDeployFailed) +} + +func (j *JobStatus) IsStopped() bool { + return j != nil && + (j.State == JobStateSucceeded || + j.State == JobStateCancelled || + j.State == JobStateFailed || + j.State == JobStateLost || + j.State == JobStateDeployFailed) +} + +func (j *JobStatus) IsTerminated(spec *JobSpec) bool { + return j.IsStopped() && !j.ShouldRestart(spec) +} + +// Check if the recorded savepoint is up to date compared to maxStateAgeToRestoreSeconds. +// If maxStateAgeToRestoreSeconds is not set, +// the savepoint is up-to-date only when the recorded savepoint is the final job state. +func (j *JobStatus) IsSavepointUpToDate(spec *JobSpec, compareTime time.Time) bool { + if j.FinalSavepoint { + return true + } + if compareTime.IsZero() || + spec.MaxStateAgeToRestoreSeconds == nil || + j.SavepointLocation == "" || + j.SavepointTime == "" { + return false + } + + var stateMaxAge = int(*spec.MaxStateAgeToRestoreSeconds) + if !hasTimeElapsed(j.SavepointTime, compareTime, stateMaxAge) { + return true + } + return false +} + +// shouldRestartJob returns true if the controller should restart failed job. +// The controller can restart the job only if there is a savepoint that is close to the end time of the job. +func (j *JobStatus) ShouldRestart(spec *JobSpec) bool { + if j == nil || !j.IsFailed() || spec == nil { + return false + } + var tc TimeConverter + var restartEnabled = spec.RestartPolicy != nil && *spec.RestartPolicy == JobRestartPolicyFromSavepointOnFailure + var jobEndTime = tc.FromString(j.EndTime) + return restartEnabled && j.IsSavepointUpToDate(spec, jobEndTime) +} + +// Return true if job is ready to proceed update. +func (j *JobStatus) UpdateReady(spec *JobSpec, observeTime time.Time) bool { + var takeSavepointOnUpdate = spec.TakeSavepointOnUpdate == nil || *spec.TakeSavepointOnUpdate + switch { + case j == nil: + fallthrough + case !isBlank(spec.FromSavepoint): + return true + case j.IsActive(): + // When job is active and takeSavepointOnUpdate is true, only after taking savepoint with final job state, + // proceed job update. + if takeSavepointOnUpdate { + if j.FinalSavepoint { + return true + } + } else if j.IsSavepointUpToDate(spec, observeTime) { + return true + } + case j.State == JobStateUpdating && !takeSavepointOnUpdate: + return true + default: + // In other cases, check if savepoint is up-to-date compared to job end time. + var tc = TimeConverter{} + var jobEndTime time.Time + if j.EndTime != "" { + jobEndTime = tc.FromString(j.EndTime) + } + if j.IsSavepointUpToDate(spec, jobEndTime) { + return true + } + } + return false +} + +func (s *SavepointStatus) IsFailed() bool { + return s != nil && (s.State == SavepointStateTriggerFailed || s.State == SavepointStateFailed) +} + +func (r *RevisionStatus) IsUpdateTriggered() bool { + return r.CurrentRevision != r.NextRevision +} + +// TimeConverter converts between time.Time and string. +type TimeConverter struct{} + +// FromString converts string to time.Time. +func (tc *TimeConverter) FromString(timeStr string) time.Time { + timestamp, err := time.Parse( + time.RFC3339, timeStr) + if err != nil { + panic(fmt.Sprintf("Failed to parse time string: %s", timeStr)) + } + return timestamp +} + +// ToString converts time.Time to string. +func (tc *TimeConverter) ToString(timestamp time.Time) string { + return timestamp.Format(time.RFC3339) +} + +// Check time has passed +func hasTimeElapsed(timeToCheckStr string, now time.Time, intervalSec int) bool { + tc := &TimeConverter{} + timeToCheck := tc.FromString(timeToCheckStr) + intervalPassedTime := timeToCheck.Add(time.Duration(int64(intervalSec) * int64(time.Second))) + if now.After(intervalPassedTime) { + return true + } + return false +} + +func isBlank(s *string) bool { + return s == nil || strings.TrimSpace(*s) == "" +} diff --git a/api/v1beta1/flinkcluster_types_util_test.go b/api/v1beta1/flinkcluster_types_util_test.go new file mode 100644 index 00000000..1052e5fa --- /dev/null +++ b/api/v1beta1/flinkcluster_types_util_test.go @@ -0,0 +1,154 @@ +/* +Copyright 2019 Google LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "gotest.tools/assert" + "testing" + "time" +) + +func TestIsSavepointUpToDate(t *testing.T) { + var tc = &TimeConverter{} + var savepointTime = time.Now() + var jobEndTime = savepointTime.Add(time.Second * 100) + var maxStateAgeToRestoreSeconds = int32(300) + + // When maxStateAgeToRestoreSeconds is not provided + var jobSpec = JobSpec{ + MaxStateAgeToRestoreSeconds: nil, + } + var jobStatus = JobStatus{ + SavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/savepoint-123", + } + var update = jobStatus.IsSavepointUpToDate(&jobSpec, jobEndTime) + assert.Equal(t, update, false) + + // Old savepoint + savepointTime = time.Now() + jobEndTime = savepointTime.Add(time.Second * 500) + jobSpec = JobSpec{ + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + jobStatus = JobStatus{ + SavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/savepoint-123", + EndTime: tc.ToString(jobEndTime), + } + update = jobStatus.IsSavepointUpToDate(&jobSpec, jobEndTime) + assert.Equal(t, update, false) + + // Fails without savepointLocation + savepointTime = time.Now() + jobEndTime = savepointTime.Add(time.Second * 100) + jobSpec = JobSpec{ + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + jobStatus = JobStatus{ + SavepointTime: tc.ToString(savepointTime), + EndTime: tc.ToString(jobEndTime), + } + update = jobStatus.IsSavepointUpToDate(&jobSpec, jobEndTime) + assert.Equal(t, update, false) + + // Up-to-date savepoint + jobEndTime = savepointTime.Add(time.Second * 100) + jobSpec = JobSpec{ + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + jobStatus = JobStatus{ + SavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/savepoint-123", + } + update = jobStatus.IsSavepointUpToDate(&jobSpec, jobEndTime) + assert.Equal(t, update, true) + + // A savepoint of the final job state. + jobSpec = JobSpec{ + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + jobStatus = JobStatus{ + FinalSavepoint: true, + } + update = jobStatus.IsSavepointUpToDate(&jobSpec, time.Time{}) + assert.Equal(t, update, true) +} + +func TestShouldRestartJob(t *testing.T) { + var tc = &TimeConverter{} + var restartOnFailure = JobRestartPolicyFromSavepointOnFailure + var neverRestart = JobRestartPolicyNever + var maxStateAgeToRestoreSeconds = int32(300) // 5 min + + // Restart with savepoint up to date + var savepointTime = time.Now() + var endTime = savepointTime.Add(time.Second * 60) // savepointTime + 1 min + var jobSpec = JobSpec{ + RestartPolicy: &restartOnFailure, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + var jobStatus = JobStatus{ + State: JobStateFailed, + SavepointLocation: "gs://my-bucket/savepoint-123", + SavepointTime: tc.ToString(savepointTime), + EndTime: tc.ToString(endTime), + } + var restart = jobStatus.ShouldRestart(&jobSpec) + assert.Equal(t, restart, true) + + // Not restart without savepoint + jobSpec = JobSpec{ + RestartPolicy: &restartOnFailure, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + jobStatus = JobStatus{ + State: JobStateFailed, + EndTime: tc.ToString(endTime), + } + restart = jobStatus.ShouldRestart(&jobSpec) + assert.Equal(t, restart, false) + + // Not restart with restartPolicy Never + jobSpec = JobSpec{ + RestartPolicy: &neverRestart, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + jobStatus = JobStatus{ + State: JobStateFailed, + SavepointLocation: "gs://my-bucket/savepoint-123", + SavepointTime: tc.ToString(savepointTime), + EndTime: tc.ToString(endTime), + } + restart = jobStatus.ShouldRestart(&jobSpec) + assert.Equal(t, restart, false) + + // Not restart with old savepoint + endTime = savepointTime.Add(time.Second * 300) // savepointTime + 5 min + jobSpec = JobSpec{ + RestartPolicy: &neverRestart, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + } + jobStatus = JobStatus{ + State: JobStateFailed, + SavepointLocation: "gs://my-bucket/savepoint-123", + SavepointTime: tc.ToString(savepointTime), + EndTime: tc.ToString(endTime), + } + restart = jobStatus.ShouldRestart(&jobSpec) + assert.Equal(t, restart, false) +} diff --git a/api/v1beta1/flinkcluster_validate.go b/api/v1beta1/flinkcluster_validate.go index 2a35388f..d7072e58 100644 --- a/api/v1beta1/flinkcluster_validate.go +++ b/api/v1beta1/flinkcluster_validate.go @@ -20,7 +20,9 @@ import ( "encoding/json" "fmt" "reflect" + "strconv" "strings" + "time" "k8s.io/apimachinery/pkg/api/resource" @@ -82,6 +84,11 @@ func (v *Validator) ValidateUpdate(old *FlinkCluster, new *FlinkCluster) error { return err } + // Skip remaining validation if no changes in spec. + if reflect.DeepEqual(new.Spec, old.Spec) { + return nil + } + cancelRequested, err := v.checkCancelRequested(old, new) if err != nil { return err @@ -120,19 +127,19 @@ func (v *Validator) checkControlAnnotations(old *FlinkCluster, new *FlinkCluster } switch newUserControl { case ControlNameJobCancel: - var jobStatus = old.Status.Components.Job + var job = old.Status.Components.Job if old.Spec.Job == nil { return fmt.Errorf(SessionClusterWarnMsg, ControlNameJobCancel, ControlAnnotation) - } else if jobStatus == nil || isJobTerminated(old.Spec.Job.RestartPolicy, jobStatus) { + } else if job == nil || job.IsTerminated(old.Spec.Job) { return fmt.Errorf(InvalidJobStateForJobCancelMsg, ControlAnnotation) } case ControlNameSavepoint: - var jobStatus = old.Status.Components.Job + var job = old.Status.Components.Job if old.Spec.Job == nil { return fmt.Errorf(SessionClusterWarnMsg, ControlNameSavepoint, ControlAnnotation) } else if old.Spec.Job.SavepointsDir == nil || *old.Spec.Job.SavepointsDir == "" { return fmt.Errorf(InvalidSavepointDirMsg, ControlAnnotation) - } else if jobStatus == nil || jobStatus.isJobStopped() { + } else if job == nil || job.IsStopped() { return fmt.Errorf(InvalidJobStateForSavepointMsg, ControlAnnotation) } default: @@ -207,28 +214,42 @@ func (v *Validator) checkSavepointGeneration( // Validate job update. func (v *Validator) validateJobUpdate(old *FlinkCluster, new *FlinkCluster) error { - var jobStatus = old.Status.Components.Job switch { case old.Spec.Job == nil && new.Spec.Job == nil: return nil case old.Spec.Job == nil || new.Spec.Job == nil: - oldJob, _ := json.Marshal(old.Spec.Job) - newJob, _ := json.Marshal(new.Spec.Job) - return fmt.Errorf("you cannot change cluster type between session cluster and job cluster, old spec.job: %q, new spec.job: %q", oldJob, newJob) + oldJobSpec, _ := json.Marshal(old.Spec.Job) + newJobSpec, _ := json.Marshal(new.Spec.Job) + return fmt.Errorf("you cannot change cluster type between session cluster and job cluster, old spec.job: %q, new spec.job: %q", oldJobSpec, newJobSpec) case old.Spec.Job.SavepointsDir == nil || *old.Spec.Job.SavepointsDir == "": return fmt.Errorf("updating job is not allowed when spec.job.savepointsDir was not provided") case old.Spec.Job.SavepointsDir != nil && *old.Spec.Job.SavepointsDir != "" && (new.Spec.Job.SavepointsDir == nil || *new.Spec.Job.SavepointsDir == ""): return fmt.Errorf("removing savepointsDir is not allowed") - case jobStatus != nil && jobStatus.isJobStopped(): - if jobStatus.FinalSavepoint { - return nil - } - var shouldTakeSavepoint = (new.Spec.Job.TakeSavepointOnUpdate == nil || *new.Spec.Job.TakeSavepointOnUpdate) && - (new.Spec.Job.FromSavepoint == nil || *new.Spec.Job.FromSavepoint == "") - if shouldTakeSavepoint { - return fmt.Errorf("cannot update because job is stoppped without final savepoint," + - "to proceed update, you need to set takeSavepointOnUpdate false or provide fromSavepoint") + case !isBlank(new.Spec.Job.FromSavepoint): + return nil + default: + // In the case of taking savepoint is skipped, check if the savepoint is up-to-date. + var oldJob = old.Status.Components.Job + var takeSavepointOnUpdate = new.Spec.Job.TakeSavepointOnUpdate == nil || *new.Spec.Job.TakeSavepointOnUpdate + var skipTakeSavepoint = !takeSavepointOnUpdate || oldJob.IsStopped() + var now = time.Now() + if skipTakeSavepoint && oldJob != nil && !oldJob.UpdateReady(new.Spec.Job, now) { + oldJobJson, _ := json.Marshal(oldJob) + var takeSP, maxStateAge string + if new.Spec.Job.TakeSavepointOnUpdate == nil { + takeSP = "nil" + } else { + takeSP = strconv.FormatBool(*new.Spec.Job.TakeSavepointOnUpdate) + } + if new.Spec.Job.MaxStateAgeToRestoreSeconds == nil { + maxStateAge = "nil" + } else { + maxStateAge = strconv.Itoa(int(*new.Spec.Job.MaxStateAgeToRestoreSeconds)) + } + return fmt.Errorf("cannot update spec: taking savepoint is skipped but no up-to-date savepoint, "+ + "spec.job.takeSavepointOnUpdate: %v, spec.job.maxStateAgeToRestoreSeconds: %v, job status: %q", + takeSP, maxStateAge, oldJobJson) } } return nil @@ -424,10 +445,18 @@ func (v *Validator) validateJob(jobSpec *JobSpec) error { switch *jobSpec.RestartPolicy { case JobRestartPolicyNever: case JobRestartPolicyFromSavepointOnFailure: + if jobSpec.MaxStateAgeToRestoreSeconds == nil { + return fmt.Errorf("maxStateAgeToRestoreSeconds must be specified when restartPolicy is set as FromSavepointOnFailure") + } default: return fmt.Errorf("invalid job restartPolicy: %v", *jobSpec.RestartPolicy) } + if jobSpec.TakeSavepointOnUpdate != nil && *jobSpec.TakeSavepointOnUpdate == false && + jobSpec.MaxStateAgeToRestoreSeconds == nil { + return fmt.Errorf("maxStateAgeToRestoreSeconds must be specified when takeSavepointOnUpdate is set as false") + } + if jobSpec.CleanupPolicy == nil { return fmt.Errorf("job cleanupPolicy is unspecified") } @@ -518,19 +547,3 @@ func (v *Validator) validateMemoryOffHeapMin( } return nil } - -// shouldRestartJob returns true if the controller should restart the failed -// job. -func shouldRestartJob( - restartPolicy *JobRestartPolicy, - jobStatus *JobStatus) bool { - return restartPolicy != nil && - *restartPolicy == JobRestartPolicyFromSavepointOnFailure && - jobStatus != nil && - jobStatus.State == JobStateFailed && - len(jobStatus.SavepointLocation) > 0 -} - -func isJobTerminated(restartPolicy *JobRestartPolicy, jobStatus *JobStatus) bool { - return jobStatus.isJobStopped() && !shouldRestartJob(restartPolicy, jobStatus) -} diff --git a/api/v1beta1/flinkcluster_validate_test.go b/api/v1beta1/flinkcluster_validate_test.go index 4319a066..9c2f5a7f 100644 --- a/api/v1beta1/flinkcluster_validate_test.go +++ b/api/v1beta1/flinkcluster_validate_test.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" "testing" + "time" "k8s.io/apimachinery/pkg/api/resource" @@ -28,6 +29,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const MaxStateAgeToRestore = int32(60) + func TestValidateCreate(t *testing.T) { var jmReplicas int32 = 1 var rpcPort int32 = 8001 @@ -36,6 +39,7 @@ func TestValidateCreate(t *testing.T) { var uiPort int32 = 8004 var dataPort int32 = 8005 var parallelism int32 = 2 + var maxStateAgeToRestoreSeconds = int32(60) var restartPolicy = JobRestartPolicyFromSavepointOnFailure var memoryOffHeapRatio int32 = 25 var memoryOffHeapMin = resource.MustParse("600M") @@ -72,9 +76,10 @@ func TestValidateCreate(t *testing.T) { MemoryOffHeapMin: memoryOffHeapMin, }, Job: &JobSpec{ - JarFile: "gs://my-bucket/myjob.jar", - Parallelism: ¶llelism, - RestartPolicy: &restartPolicy, + JarFile: "gs://my-bucket/myjob.jar", + Parallelism: ¶llelism, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, + RestartPolicy: &restartPolicy, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: CleanupActionKeepCluster, AfterJobFails: CleanupActionDeleteTaskManager, @@ -353,6 +358,7 @@ func TestInvalidJobSpec(t *testing.T) { var queryPort int32 = 8003 var uiPort int32 = 8004 var dataPort int32 = 8005 + var maxStateAgeToRestoreSeconds int32 = 300 var restartPolicy = JobRestartPolicyFromSavepointOnFailure var invalidRestartPolicy JobRestartPolicy = "XXX" var validator = &Validator{} @@ -393,8 +399,9 @@ func TestInvalidJobSpec(t *testing.T) { MemoryOffHeapMin: memoryOffHeapMin, }, Job: &JobSpec{ - JarFile: "", - RestartPolicy: &restartPolicy, + JarFile: "", + RestartPolicy: &restartPolicy, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, }, }, } @@ -435,8 +442,9 @@ func TestInvalidJobSpec(t *testing.T) { MemoryOffHeapMin: memoryOffHeapMin, }, Job: &JobSpec{ - JarFile: "gs://my-bucket/myjob.jar", - RestartPolicy: &restartPolicy, + JarFile: "gs://my-bucket/myjob.jar", + RestartPolicy: &restartPolicy, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, }, }, } @@ -477,9 +485,10 @@ func TestInvalidJobSpec(t *testing.T) { MemoryOffHeapMin: memoryOffHeapMin, }, Job: &JobSpec{ - JarFile: "gs://my-bucket/myjob.jar", - Parallelism: ¶llelism, - RestartPolicy: &invalidRestartPolicy, + JarFile: "gs://my-bucket/myjob.jar", + Parallelism: ¶llelism, + RestartPolicy: &invalidRestartPolicy, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, }, }, } @@ -520,9 +529,10 @@ func TestInvalidJobSpec(t *testing.T) { MemoryOffHeapMin: memoryOffHeapMin, }, Job: &JobSpec{ - JarFile: "gs://my-bucket/myjob.jar", - Parallelism: ¶llelism, - RestartPolicy: &restartPolicy, + JarFile: "gs://my-bucket/myjob.jar", + Parallelism: ¶llelism, + RestartPolicy: &restartPolicy, + MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: "XXX", AfterJobFails: CleanupActionDeleteCluster, @@ -618,47 +628,19 @@ func TestUpdateSavepointGeneration(t *testing.T) { func TestUpdateJob(t *testing.T) { var validator = &Validator{} - var parallelism int32 = 2 - var restartPolicy = JobRestartPolicyFromSavepointOnFailure - var savepointDir = "/savepoint_dir" + var tc = &TimeConverter{} + var maxStateAge = time.Duration(MaxStateAgeToRestore) - oldCluster := getSimpleFlinkCluster() - oldCluster.Spec.Job = &JobSpec{ - JarFile: "gs://my-bucket/myjob.jar", - Parallelism: ¶llelism, - RestartPolicy: &restartPolicy, - SavepointsDir: &savepointDir, - CleanupPolicy: &CleanupPolicy{ - AfterJobSucceeds: CleanupActionKeepCluster, - AfterJobFails: CleanupActionDeleteTaskManager, - }, - } - newCluster := getSimpleFlinkCluster() - newCluster.Spec.Job = &JobSpec{ - JarFile: "gs://my-bucket/myjob.jar", - Parallelism: ¶llelism, - RestartPolicy: &restartPolicy, - SavepointsDir: nil, - CleanupPolicy: &CleanupPolicy{ - AfterJobSucceeds: CleanupActionKeepCluster, - AfterJobFails: CleanupActionDeleteTaskManager, - }, - } + // cannot remove savepointsDir + var oldCluster = getSimpleFlinkCluster() + var newCluster = getSimpleFlinkCluster() + newCluster.Spec.Job.SavepointsDir = nil err := validator.ValidateUpdate(&oldCluster, &newCluster) expectedErr := "removing savepointsDir is not allowed" assert.Equal(t, err.Error(), expectedErr) + // cannot change cluster type oldCluster = getSimpleFlinkCluster() - oldCluster.Spec.Job = &JobSpec{ - JarFile: "gs://my-bucket/myjob.jar", - Parallelism: ¶llelism, - RestartPolicy: &restartPolicy, - SavepointsDir: &savepointDir, - CleanupPolicy: &CleanupPolicy{ - AfterJobSucceeds: CleanupActionKeepCluster, - AfterJobFails: CleanupActionDeleteTaskManager, - }, - } newCluster = getSimpleFlinkCluster() newCluster.Spec.Job = nil err = validator.ValidateUpdate(&oldCluster, &newCluster) @@ -667,29 +649,111 @@ func TestUpdateJob(t *testing.T) { expectedErr = fmt.Sprintf("you cannot change cluster type between session cluster and job cluster, old spec.job: %q, new spec.job: %q", oldJson, newJson) assert.Equal(t, err.Error(), expectedErr) + // cannot update when savepointDir is not provided oldCluster = getSimpleFlinkCluster() - oldCluster.Spec.Job = &JobSpec{ - JarFile: "gs://my-bucket/myjob-v1.jar", - Parallelism: ¶llelism, - RestartPolicy: &restartPolicy, - CleanupPolicy: &CleanupPolicy{ - AfterJobSucceeds: CleanupActionKeepCluster, - AfterJobFails: CleanupActionDeleteTaskManager, - }, + oldCluster.Spec.Job.SavepointsDir = nil + newCluster = getSimpleFlinkCluster() + newCluster.Spec.Job.SavepointsDir = nil + newCluster.Spec.Job.JarFile = "gs://my-bucket/myjob-v2.jar" + err = validator.ValidateUpdate(&oldCluster, &newCluster) + expectedErr = "updating job is not allowed when spec.job.savepointsDir was not provided" + assert.Equal(t, err.Error(), expectedErr) + + // cannot update when takeSavepointOnUpdate is false and stale savepoint + var takeSavepointOnUpdateFalse = false + var savepointTime = time.Now().Add(-(maxStateAge + 10) * time.Second) // stale savepoint + oldCluster = getSimpleFlinkCluster() + oldCluster.Status.Components.Job = &JobStatus{ + SavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/my-sp-123", + State: JobStateRunning, } newCluster = getSimpleFlinkCluster() - newCluster.Spec.Job = &JobSpec{ - JarFile: "gs://my-bucket/myjob-v2.jar", - Parallelism: ¶llelism, - RestartPolicy: &restartPolicy, - CleanupPolicy: &CleanupPolicy{ - AfterJobSucceeds: CleanupActionKeepCluster, - AfterJobFails: CleanupActionDeleteTaskManager, - }, + newCluster.Spec.Job.JarFile = "gs://my-bucket/myjob-v2.jar" + newCluster.Spec.Job.TakeSavepointOnUpdate = &takeSavepointOnUpdateFalse + err = validator.ValidateUpdate(&oldCluster, &newCluster) + jobStatusJson, _ := json.Marshal(oldCluster.Status.Components.Job) + expectedErr = fmt.Sprintf("cannot update spec: taking savepoint is skipped but no up-to-date savepoint, "+ + "spec.job.takeSavepointOnUpdate: false, spec.job.maxStateAgeToRestoreSeconds: 60, job status: %q", jobStatusJson) + assert.Equal(t, err.Error(), expectedErr) + + // update when takeSavepointOnUpdate is false and savepoint is up-to-date + takeSavepointOnUpdateFalse = false + maxStateAge = time.Duration(*getSimpleFlinkCluster().Spec.Job.MaxStateAgeToRestoreSeconds) + savepointTime = time.Now().Add(-(maxStateAge - 10) * time.Second) // up-to-date savepoint + oldCluster = getSimpleFlinkCluster() + oldCluster.Status.Components.Job = &JobStatus{ + SavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/my-sp-123", + State: JobStateRunning, } + newCluster = getSimpleFlinkCluster() + newCluster.Spec.Job.JarFile = "gs://my-bucket/myjob-v2.jar" + newCluster.Spec.Job.TakeSavepointOnUpdate = &takeSavepointOnUpdateFalse err = validator.ValidateUpdate(&oldCluster, &newCluster) - expectedErr = "updating job is not allowed when spec.job.savepointsDir was not provided" + assert.Equal(t, err, nil) + + // spec update is allowed when takeSavepointOnUpdate is true and savepoint is not completed yet + oldCluster = getSimpleFlinkCluster() + oldCluster.Status.Components.Job = &JobStatus{ + FinalSavepoint: false, + SavepointLocation: "gs://my-bucket/my-sp-123", + State: JobStateRunning, + } + newCluster = getSimpleFlinkCluster() + newCluster.Spec.Job.JarFile = "gs://my-bucket/myjob-v2.jar" + err = validator.ValidateUpdate(&oldCluster, &newCluster) + assert.Equal(t, err, nil) + + // when job is stopped and no up-to-date savepoint + var jobEndTime = time.Now() + savepointTime = jobEndTime.Add(-(maxStateAge + 10) * time.Second) // stale savepoint + oldCluster = getSimpleFlinkCluster() + oldCluster.Status.Components.Job = &JobStatus{ + SavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/my-sp-123", + State: JobStateFailed, + EndTime: tc.ToString(jobEndTime), + } + newCluster = getSimpleFlinkCluster() + newCluster.Spec.Job.JarFile = "gs://my-bucket/myjob-v2.jar" + err = validator.ValidateUpdate(&oldCluster, &newCluster) + jobStatusJson, _ = json.Marshal(oldCluster.Status.Components.Job) + expectedErr = fmt.Sprintf("cannot update spec: taking savepoint is skipped but no up-to-date savepoint, "+ + "spec.job.takeSavepointOnUpdate: nil, spec.job.maxStateAgeToRestoreSeconds: 60, job status: %q", jobStatusJson) assert.Equal(t, err.Error(), expectedErr) + + // when job is stopped and savepoint is up-to-date + jobEndTime = time.Now() + savepointTime = jobEndTime.Add(-(maxStateAge - 10) * time.Second) // up-to-date savepoint + oldCluster = getSimpleFlinkCluster() + oldCluster.Status.Components.Job = &JobStatus{ + SavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/my-sp-123", + State: JobStateFailed, + EndTime: tc.ToString(jobEndTime), + } + newCluster = getSimpleFlinkCluster() + newCluster.Spec.Job.JarFile = "gs://my-bucket/myjob-v2.jar" + err = validator.ValidateUpdate(&oldCluster, &newCluster) + assert.Equal(t, err, nil) + + // when job is stopped and savepoint is stale, but fromSavepoint is provided + var fromSavepoint = "gs://my-bucket/sp-123" + jobEndTime = time.Now() + savepointTime = jobEndTime.Add(-(maxStateAge + 10) * time.Second) // stale savepoint + oldCluster = getSimpleFlinkCluster() + oldCluster.Status.Components.Job = &JobStatus{ + SavepointTime: tc.ToString(savepointTime), + SavepointLocation: "gs://my-bucket/my-sp-123", + State: JobStateFailed, + EndTime: tc.ToString(jobEndTime), + } + newCluster = getSimpleFlinkCluster() + newCluster.Spec.Job.JarFile = "gs://my-bucket/myjob-v2.jar" + newCluster.Spec.Job.FromSavepoint = &fromSavepoint + err = validator.ValidateUpdate(&oldCluster, &newCluster) + assert.Equal(t, err, nil) } func TestUpdateCluster(t *testing.T) { @@ -852,6 +916,7 @@ func TestUserControlSavepoint(t *testing.T) { } func TestUserControlJobCancel(t *testing.T) { + var tc = TimeConverter{} var validator = &Validator{} var restartPolicy = JobRestartPolicyNever var newCluster = FlinkCluster{ @@ -881,16 +946,22 @@ func TestUserControlJobCancel(t *testing.T) { assert.Equal(t, err3.Error(), expectedErr3) var oldCluster4 = FlinkCluster{ - Spec: FlinkClusterSpec{Job: &JobSpec{}}, - Status: FlinkClusterStatus{Components: FlinkClusterComponentsStatus{Job: &JobStatus{State: JobStateSucceeded}}}, + Spec: FlinkClusterSpec{Job: &JobSpec{}}, + Status: FlinkClusterStatus{Components: FlinkClusterComponentsStatus{Job: &JobStatus{ + State: JobStateSucceeded, + EndTime: tc.ToString(time.Now()), + }}}, } var err4 = validator.ValidateUpdate(&oldCluster4, &newCluster) var expectedErr4 = "job-cancel is not allowed because job is not started yet or already terminated, annotation: flinkclusters.flinkoperator.k8s.io/user-control" assert.Equal(t, err4.Error(), expectedErr4) var oldCluster5 = FlinkCluster{ - Spec: FlinkClusterSpec{Job: &JobSpec{RestartPolicy: &restartPolicy}}, - Status: FlinkClusterStatus{Components: FlinkClusterComponentsStatus{Job: &JobStatus{State: JobStateFailed}}}, + Spec: FlinkClusterSpec{Job: &JobSpec{RestartPolicy: &restartPolicy}}, + Status: FlinkClusterStatus{Components: FlinkClusterComponentsStatus{ + Job: &JobStatus{State: JobStateFailed, + EndTime: tc.ToString(time.Now()), + }}}, } var err5 = validator.ValidateUpdate(&oldCluster5, &newCluster) var expectedErr5 = "job-cancel is not allowed because job is not started yet or already terminated, annotation: flinkclusters.flinkoperator.k8s.io/user-control" @@ -965,6 +1036,7 @@ func getSimpleFlinkCluster() FlinkCluster { var memoryOffHeapRatio int32 = 25 var memoryOffHeapMin = resource.MustParse("600M") var parallelism int32 = 2 + var maxStateAge = MaxStateAgeToRestore var restartPolicy = JobRestartPolicyFromSavepointOnFailure var savepointDir = "/savepoint_dir" return FlinkCluster{ @@ -1000,10 +1072,11 @@ func getSimpleFlinkCluster() FlinkCluster { MemoryOffHeapMin: memoryOffHeapMin, }, Job: &JobSpec{ - JarFile: "gs://my-bucket/myjob.jar", - Parallelism: ¶llelism, - RestartPolicy: &restartPolicy, - SavepointsDir: &savepointDir, + JarFile: "gs://my-bucket/myjob.jar", + Parallelism: ¶llelism, + MaxStateAgeToRestoreSeconds: &maxStateAge, + RestartPolicy: &restartPolicy, + SavepointsDir: &savepointDir, CleanupPolicy: &CleanupPolicy{ AfterJobSucceeds: CleanupActionKeepCluster, AfterJobFails: CleanupActionDeleteTaskManager, diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 93e8f5ae..25b579e8 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -868,6 +868,21 @@ func (in *TaskManagerSpec) DeepCopy() *TaskManagerSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TimeConverter) DeepCopyInto(out *TimeConverter) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimeConverter. +func (in *TimeConverter) DeepCopy() *TimeConverter { + if in == nil { + return nil + } + out := new(TimeConverter) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Validator) DeepCopyInto(out *Validator) { *out = *in diff --git a/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml b/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml index 36e67f03..f0474f3e 100644 --- a/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml +++ b/config/crd/bases/flinkoperator.k8s.io_flinkclusters.yaml @@ -716,6 +716,7 @@ spec: type: string maxStateAgeToRestoreSeconds: format: int32 + minimum: 0 type: integer noLoggingToStdout: type: boolean diff --git a/controllers/flinkcluster_converter.go b/controllers/flinkcluster_converter.go index fa474313..96cb42de 100644 --- a/controllers/flinkcluster_converter.go +++ b/controllers/flinkcluster_converter.go @@ -603,8 +603,8 @@ func getDesiredJob(observed *ObservedClusterState) *batchv1.Job { } // When the job should be stopped, keep that state unless update is triggered or the job must to be restarted. - if (shouldStopJob(flinkCluster) || isJobStopped(jobStatus)) && - !(observed.isClusterUpdating() || shouldRestartJob(jobSpec, jobStatus)) { + if (shouldStopJob(flinkCluster) || jobStatus.IsStopped()) && + !(shouldUpdateJob(observed) || jobStatus.ShouldRestart(jobSpec)) { return nil } @@ -779,7 +779,7 @@ func convertFromSavepoint(jobSpec *v1beta1.JobSpec, jobStatus *v1beta1.JobStatus } return nil // Updating with FromSavepoint provided - case isUpdateTriggered(revision) && !isBlank(jobSpec.FromSavepoint): + case revision.IsUpdateTriggered() && !isBlank(jobSpec.FromSavepoint): return jobSpec.FromSavepoint // Latest savepoint case jobStatus.SavepointLocation != "": @@ -872,7 +872,7 @@ func shouldCleanup( return false } - if isUpdateTriggered(&cluster.Status.Revision) { + if cluster.Status.Revision.IsUpdateTriggered() { return false } diff --git a/controllers/flinkcluster_observer.go b/controllers/flinkcluster_observer.go index 04ebfa9c..adf84fe0 100644 --- a/controllers/flinkcluster_observer.go +++ b/controllers/flinkcluster_observer.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "errors" "fmt" "github.com/go-logr/logr" v1beta1 "github.com/googlecloudplatform/flink-operator/api/v1beta1" @@ -96,10 +95,6 @@ func (o *ObservedClusterState) isClusterUpdating() bool { return o.updateState == UpdateStateInProgress } -func (r *Revision) isUpdateTriggered() bool { - return getRevisionWithNameNumber(r.currentRevision) != getRevisionWithNameNumber(r.nextRevision) -} - // Job submitter status. func (s *FlinkJobSubmitter) getState() JobSubmitState { switch { @@ -334,17 +329,19 @@ func (observer *ClusterStateObserver) observeSubmitter(submitter *FlinkJobSubmit } submitter.pod = pod - // Extract submit result. + // Extract submission result. var jobSubmissionCompleted = job.Status.Succeeded > 0 || job.Status.Failed > 0 if !jobSubmissionCompleted { return nil } log.Info("Extracting the result of job submission because it is completed") podLog = new(SubmitterLog) - err = observeFlinkJobSubmitterLog(pod, podLog) + err = observer.observeFlinkJobSubmitterLog(pod, podLog) if err != nil { log.Error(err, "Failed to extract the job submission result") podLog = nil + } else if podLog == nil { + log.Info("Observed submitter log", "state", "nil") } else { log.Info("Observed submitter log", "state", *podLog) } @@ -389,10 +386,7 @@ func (observer *ClusterStateObserver) observeFlinkJobStatus( } flinkJob.list = flinkJobList - // Extract the current job status and unexpected jobs, if submitted job ID is provided. - if flinkJobID == "" { - return - } + // Extract the current job status and unexpected jobs. for _, job := range flinkJobList.Jobs { if flinkJobID == job.ID { flinkJobStatus = &job @@ -402,19 +396,12 @@ func (observer *ClusterStateObserver) observeFlinkJobStatus( } flinkJob.status = flinkJobStatus flinkJob.unexpected = flinkJobsUnexpected - - // It is okay if there are multiple jobs, but at most one of them is - // expected to be running. This is typically caused by job client - // timed out and exited but the job submission was actually - // successfully. When retrying, it first cancels the existing running - // job which it has lost track of, then submit the job again. - if len(flinkJobsUnexpected) > 1 { - log.Error( - errors.New("more than one unexpected Flink job were found"), - "", "unexpected jobs", flinkJobsUnexpected) - } - if flinkJob != nil { - log.Info("Observed Flink job", "flink job", flinkJob) + log.Info("Observed Flink job", + "submitted job status", flinkJob.status, + "all job list", flinkJob.list, + "unexpected job list", flinkJob.unexpected) + if len(flinkJobsUnexpected) > 0 { + log.Info("More than one unexpected Flink job were found!") } return @@ -695,12 +682,15 @@ func (observer *ClusterStateObserver) truncateHistory(observed *ObservedClusterS } // observeFlinkJobSubmit extract submit result from the pod termination log. -func observeFlinkJobSubmitterLog(observedPod *corev1.Pod, submitterLog *SubmitterLog) error { +func (observer *ClusterStateObserver) observeFlinkJobSubmitterLog(observedPod *corev1.Pod, submitterLog *SubmitterLog) error { + var log = observer.log var containerStatuses = observedPod.Status.ContainerStatuses if len(containerStatuses) == 0 || containerStatuses[0].State.Terminated == nil || containerStatuses[0].State.Terminated.Message == "" { - return fmt.Errorf("job pod found, but no termination log") + submitterLog = nil + log.Info("job pod found, but no termination log") + return nil } // The job submission script writes the submission log to the pod termination log at the end of execution. diff --git a/controllers/flinkcluster_observer_test.go b/controllers/flinkcluster_observer_test.go index c1b85335..3a03b5d4 100644 --- a/controllers/flinkcluster_observer_test.go +++ b/controllers/flinkcluster_observer_test.go @@ -25,8 +25,8 @@ import ( func TestGetFlinkJobDeploymentState(t *testing.T) { var pod corev1.Pod var submitterLog, expected *SubmitterLog - var err error var termMsg string + var observer = ClusterStateObserver{} // success termMsg = ` @@ -55,17 +55,6 @@ Job has been submitted with JobID ec74209eb4e3db8ae72db00bd7a830aa Message: termMsg, }}}}}} submitterLog = new(SubmitterLog) - _ = observeFlinkJobSubmitterLog(&pod, submitterLog) + _ = observer.observeFlinkJobSubmitterLog(&pod, submitterLog) assert.DeepEqual(t, *submitterLog, *expected) - - // failed: message not found - pod = corev1.Pod{ - Status: corev1.PodStatus{ - ContainerStatuses: []corev1.ContainerStatus{{ - State: corev1.ContainerState{ - Terminated: &corev1.ContainerStateTerminated{ - Message: "", - }}}}}} - err = observeFlinkJobSubmitterLog(&pod, submitterLog) - assert.Error(t, err, "job pod found, but no termination log") } diff --git a/controllers/flinkcluster_reconciler.go b/controllers/flinkcluster_reconciler.go index 5d37ba55..06eef321 100644 --- a/controllers/flinkcluster_reconciler.go +++ b/controllers/flinkcluster_reconciler.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "errors" "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -52,7 +53,9 @@ type ClusterReconciler struct { recorder record.EventRecorder } -var requeueResult = ctrl.Result{RequeueAfter: 10 * time.Second, Requeue: true} +const JobCheckInterval = 10 * time.Second + +var requeueResult = ctrl.Result{RequeueAfter: JobCheckInterval, Requeue: true} // Compares the desired state and the observed state, if there is a difference, // takes actions to drive the observed state towards the desired state. @@ -65,7 +68,7 @@ func (reconciler *ClusterReconciler) reconcile() (ctrl.Result, error) { return ctrl.Result{}, nil } - if reconciler.observed.isClusterUpdating() { + if shouldUpdateCluster(&reconciler.observed) { reconciler.log.Info("The cluster update is in progress") } // If batch-scheduling enabled @@ -138,7 +141,7 @@ func (reconciler *ClusterReconciler) reconcileStatefulSet( if desiredStatefulSet != nil && observedStatefulSet != nil { var cluster = reconciler.observed.cluster - if reconciler.observed.isClusterUpdating() && !isComponentUpdated(observedStatefulSet, cluster) { + if shouldUpdateCluster(&reconciler.observed) && !isComponentUpdated(observedStatefulSet, cluster) { updateComponent := fmt.Sprintf("%v StatefulSet", component) var err error if *reconciler.observed.cluster.Spec.RecreateOnUpdate { @@ -255,7 +258,7 @@ func (reconciler *ClusterReconciler) reconcileJobManagerService() error { if desiredJmService != nil && observedJmService != nil { var cluster = reconciler.observed.cluster - if reconciler.observed.isClusterUpdating() && !isComponentUpdated(observedJmService, cluster) { + if shouldUpdateCluster(&reconciler.observed) && !isComponentUpdated(observedJmService, cluster) { // v1.Service API does not handle update correctly when below values are empty. desiredJmService.SetResourceVersion(observedJmService.GetResourceVersion()) desiredJmService.Spec.ClusterIP = observedJmService.Spec.ClusterIP @@ -324,7 +327,7 @@ func (reconciler *ClusterReconciler) reconcileJobManagerIngress() error { if desiredJmIngress != nil && observedJmIngress != nil { var cluster = reconciler.observed.cluster - if reconciler.observed.isClusterUpdating() && !isComponentUpdated(observedJmIngress, cluster) { + if shouldUpdateCluster(&reconciler.observed) && !isComponentUpdated(observedJmIngress, cluster) { var err error if *reconciler.observed.cluster.Spec.RecreateOnUpdate { err = reconciler.deleteOldComponent(desiredJmIngress, observedJmIngress, "JobManager ingress") @@ -390,7 +393,7 @@ func (reconciler *ClusterReconciler) reconcileConfigMap() error { if desiredConfigMap != nil && observedConfigMap != nil { var cluster = reconciler.observed.cluster - if reconciler.observed.isClusterUpdating() && !isComponentUpdated(observedConfigMap, cluster) { + if shouldUpdateCluster(&reconciler.observed) && !isComponentUpdated(observedConfigMap, cluster) { var err error if *reconciler.observed.cluster.Spec.RecreateOnUpdate { err = reconciler.deleteOldComponent(desiredConfigMap, observedConfigMap, "ConfigMap") @@ -452,8 +455,7 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { var observed = reconciler.observed var recorded = observed.cluster.Status var jobSpec = observed.cluster.Spec.Job - var jobStatus = recorded.Components.Job - var activeFlinkJob bool + var job = recorded.Components.Job var err error var jobID = reconciler.getFlinkJobID() @@ -462,28 +464,21 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { var newControlStatus *v1beta1.FlinkClusterControlStatus defer reconciler.updateStatus(&newSavepointStatus, &newControlStatus) - // Check if Flink job is active - if isJobActive(jobStatus) { - activeFlinkJob = true - } else { - activeFlinkJob = false - } - // Create new Flink job submitter when starting new job, updating job or restarting job in failure. - if desiredJob != nil && !activeFlinkJob { + if desiredJob != nil && !job.IsActive() { log.Info("Deploying Flink job") + // TODO: Record event or introduce Condition in CRD status to notify update state pended. + // https://github.com/kubernetes/apimachinery/blob/57f2a0733447cfd41294477d833cce6580faaca3/pkg/apis/meta/v1/types.go#L1376 var unexpectedJobs = observed.flinkJob.unexpected if len(unexpectedJobs) > 0 { - if jobSpec.MaxStateAgeToRestoreSeconds != nil { - log.Info("Cancelling unexpected running job(s)") - err = reconciler.cancelUnexpectedJobs(false /* takeSavepoint */) - return requeueResult, err - } - // In this case the user should identify the cause of the problem - // so that the job is not accidentally executed multiple times by mistake or the Flink operator's error. - err = fmt.Errorf("failed to create job submitter, unexpected jobs found: %v", unexpectedJobs) - return ctrl.Result{}, err + // This is an exceptional situation. + // There should be no jobs because all jobs are terminated in the previous iterations. + // In this case user should identify the problem so that the job is not executed multiple times unintentionally + // cause of Flink error, Flink operator error or other unknown error. + // If user want to proceed, unexpected jobs should be terminated. + log.Error(errors.New("unexpected jobs found"), "Failed to create job submitter", "unexpected jobs", unexpectedJobs) + return ctrl.Result{}, nil } // Create Flink job submitter @@ -507,17 +502,20 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { return requeueResult, err } - if desiredJob != nil && activeFlinkJob { - if jobStatus.State == v1beta1.JobStateDeploying { - log.Info("Job submitter is deployed and wait until it is completed") + if desiredJob != nil && job.IsActive() { + if job.State == v1beta1.JobStateDeploying { + log.Info("Job submitter is deployed, wait until completed") return requeueResult, nil } - if isUpdateTriggered(&recorded.Revision) { + + // Suspend or stop job to proceed update. + if recorded.Revision.IsUpdateTriggered() { log.Info("Preparing job update") - var shouldSuspend = jobSpec.TakeSavepointOnUpdate == nil || *jobSpec.TakeSavepointOnUpdate == true || isBlank(jobSpec.FromSavepoint) + var takeSavepoint = jobSpec.TakeSavepointOnUpdate == nil || *jobSpec.TakeSavepointOnUpdate + var shouldSuspend = takeSavepoint && isBlank(jobSpec.FromSavepoint) if shouldSuspend { newSavepointStatus, err = reconciler.trySuspendJob() - } else { + } else if shouldUpdateJob(&observed) { err = reconciler.cancelJob() } return requeueResult, err @@ -542,7 +540,7 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { } // Job cancel requested. Stop Flink job. - if desiredJob == nil && activeFlinkJob { + if desiredJob == nil && job.IsActive() { log.Info("Stopping job", "jobID", jobID) newSavepointStatus, err = reconciler.trySuspendJob() var userControl = getNewControlRequest(observed.cluster) @@ -552,7 +550,7 @@ func (reconciler *ClusterReconciler) reconcileJob() (ctrl.Result, error) { return requeueResult, err } - if isJobStopped(jobStatus) { + if job.IsStopped() { log.Info("Job has finished, no action") } @@ -633,7 +631,7 @@ func (reconciler *ClusterReconciler) cancelJob() error { return err } - // TODO: It would be nice not to delete the job submitters immediately, and retain the latest ones for debug. + // TODO: Not to delete the job submitter immediately, and retain the latest ones for inspection. var observedSubmitter = reconciler.observed.flinkJobSubmitter.job if observedSubmitter != nil { var err = reconciler.deleteJob(observedSubmitter) @@ -723,7 +721,7 @@ func (reconciler *ClusterReconciler) canSuspendJob(jobID string, s *v1beta1.Save log.Info("Savepoint failed on previous request") } - var retryTimeArrived = hasTimeElapsed(s.UpdateTime, time.Now(), SavepointRequestRetryIntervalSec) + var retryTimeArrived = hasTimeElapsed(s.UpdateTime, time.Now(), SavepointRetryIntervalSeconds) if !retryTimeArrived { log.Info("Wait until next retry time arrived") } @@ -734,7 +732,8 @@ func (reconciler *ClusterReconciler) shouldTakeSavepoint() string { var observed = reconciler.observed var cluster = observed.cluster var jobSpec = observed.cluster.Spec.Job - var jobStatus = observed.cluster.Status.Components.Job + var job = observed.cluster.Status.Components.Job + var savepoint = observed.cluster.Status.Savepoint var newRequestedControl = getNewControlRequest(cluster) if !canTakeSavepoint(*reconciler.observed.cluster) { @@ -747,11 +746,11 @@ func (reconciler *ClusterReconciler) shouldTakeSavepoint() string { // Should stop job with savepoint by user requested control case newRequestedControl == v1beta1.ControlNameJobCancel || (jobSpec.CancelRequested != nil && *jobSpec.CancelRequested): return v1beta1.SavepointTriggerReasonJobCancel - // TODO: spec.job.savepointGeneration will be deprecated // Take savepoint by user request case newRequestedControl == v1beta1.ControlNameSavepoint: fallthrough - case jobSpec.SavepointGeneration > jobStatus.SavepointGeneration: + // TODO: spec.job.savepointGeneration will be deprecated + case jobSpec.SavepointGeneration > job.SavepointGeneration: // Triggered by savepointGeneration increased. // When previous savepoint is failed, savepoint trigger by spec.job.savepointGeneration is not possible // because the field cannot be increased more. @@ -759,19 +758,27 @@ func (reconciler *ClusterReconciler) shouldTakeSavepoint() string { return v1beta1.SavepointTriggerReasonUserRequested // Scheduled auto savepoint case jobSpec.AutoSavepointSeconds != nil: + // When previous try was failed, check retry interval. + if savepoint.IsFailed() && savepoint.TriggerReason == v1beta1.SavepointTriggerReasonScheduled { + var nextRetryTime = getTime(savepoint.UpdateTime).Add(SavepointRetryIntervalSeconds * time.Second) + if time.Now().After(nextRetryTime) { + return v1beta1.SavepointTriggerReasonScheduled + } else { + return "" + } + } // Check if next trigger time arrived. var compareTime string - if len(jobStatus.SavepointTime) == 0 { - compareTime = jobStatus.StartTime + if len(job.SavepointTime) == 0 { + compareTime = job.StartTime } else { - compareTime = jobStatus.SavepointTime + compareTime = job.SavepointTime } var nextTime = getTimeAfterAddedSeconds(compareTime, int64(*jobSpec.AutoSavepointSeconds)) if time.Now().After(nextTime) { return v1beta1.SavepointTriggerReasonScheduled } } - return "" } @@ -827,7 +834,8 @@ func (reconciler *ClusterReconciler) takeSavepoint( return err } -func (reconciler *ClusterReconciler) updateStatus(ss **v1beta1.SavepointStatus, cs **v1beta1.FlinkClusterControlStatus) { +func (reconciler *ClusterReconciler) updateStatus( + ss **v1beta1.SavepointStatus, cs **v1beta1.FlinkClusterControlStatus) { var log = reconciler.log var savepointStatus = *ss @@ -885,16 +893,24 @@ func (reconciler *ClusterReconciler) updateJobDeployStatus() error { var clusterClone = observedCluster.DeepCopy() var newJob = clusterClone.Status.Components.Job + // Reset running job information. + newJob.ID = "" + newJob.StartTime = "" + newJob.EndTime = "" + + // Mark as job submitter is deployed. + setTimestamp(&newJob.DeployTime) + setTimestamp(&clusterClone.Status.LastUpdateTime) + // Latest savepoint location should be fromSavepoint. var fromSavepoint = getFromSavepoint(desiredJobSubmitter.Spec) newJob.FromSavepoint = fromSavepoint if newJob.SavepointLocation != "" { newJob.SavepointLocation = fromSavepoint } - setTimestamp(&newJob.DeployTime) // Mark as job submitter is deployed. - setTimestamp(&clusterClone.Status.LastUpdateTime) - err = reconciler.k8sClient.Status().Update(reconciler.context, clusterClone) + // Update job status. + err = reconciler.k8sClient.Status().Update(reconciler.context, clusterClone) if err != nil { log.Error( err, "Failed to update job status for new job submitter", "error", err) diff --git a/controllers/flinkcluster_updater.go b/controllers/flinkcluster_updater.go index 0e15fe76..18aedba3 100644 --- a/controllers/flinkcluster_updater.go +++ b/controllers/flinkcluster_updater.go @@ -195,7 +195,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( // ConfigMap. var observedConfigMap = observed.configMap - if !isComponentUpdated(observedConfigMap, observed.cluster) && observed.isClusterUpdating() { + if !isComponentUpdated(observedConfigMap, observed.cluster) && shouldUpdateCluster(observed) { recorded.Components.ConfigMap.DeepCopyInto(&status.Components.ConfigMap) status.Components.ConfigMap.State = v1beta1.ComponentStateUpdating } else if observedConfigMap != nil { @@ -211,7 +211,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( // JobManager StatefulSet. var observedJmStatefulSet = observed.jmStatefulSet - if !isComponentUpdated(observedJmStatefulSet, observed.cluster) && observed.isClusterUpdating() { + if !isComponentUpdated(observedJmStatefulSet, observed.cluster) && shouldUpdateCluster(observed) { recorded.Components.JobManagerStatefulSet.DeepCopyInto(&status.Components.JobManagerStatefulSet) status.Components.JobManagerStatefulSet.State = v1beta1.ComponentStateUpdating } else if observedJmStatefulSet != nil { @@ -230,7 +230,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( // JobManager service. var observedJmService = observed.jmService - if !isComponentUpdated(observedJmService, observed.cluster) && observed.isClusterUpdating() { + if !isComponentUpdated(observedJmService, observed.cluster) && shouldUpdateCluster(observed) { recorded.Components.JobManagerService.DeepCopyInto(&status.Components.JobManagerService) status.Components.JobManagerService.State = v1beta1.ComponentStateUpdating } else if observedJmService != nil { @@ -281,7 +281,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( // (Optional) JobManager ingress. var observedJmIngress = observed.jmIngress - if !isComponentUpdated(observedJmIngress, observed.cluster) && observed.isClusterUpdating() { + if !isComponentUpdated(observedJmIngress, observed.cluster) && shouldUpdateCluster(observed) { status.Components.JobManagerIngress = &v1beta1.JobManagerIngressStatus{} recorded.Components.JobManagerIngress.DeepCopyInto(status.Components.JobManagerIngress) status.Components.JobManagerIngress.State = v1beta1.ComponentStateUpdating @@ -364,7 +364,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( // TaskManager StatefulSet. var observedTmStatefulSet = observed.tmStatefulSet - if !isComponentUpdated(observedTmStatefulSet, observed.cluster) && observed.isClusterUpdating() { + if !isComponentUpdated(observedTmStatefulSet, observed.cluster) && shouldUpdateCluster(observed) { recorded.Components.TaskManagerStatefulSet.DeepCopyInto(&status.Components.TaskManagerStatefulSet) status.Components.TaskManagerStatefulSet.State = v1beta1.ComponentStateUpdating } else if observedTmStatefulSet != nil { @@ -393,10 +393,10 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( status.State = v1beta1.ClusterStateRunning } case v1beta1.ClusterStateUpdating: - if observed.isClusterUpdating() { + if shouldUpdateCluster(observed) { status.State = v1beta1.ClusterStateUpdating } else if runningComponents < totalComponents { - if isUpdateTriggered(&recorded.Revision) { + if recorded.Revision.IsUpdateTriggered() { status.State = v1beta1.ClusterStateUpdating } else { status.State = v1beta1.ClusterStateReconciling @@ -407,9 +407,9 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( case v1beta1.ClusterStateRunning, v1beta1.ClusterStateReconciling: var jobStatus = recorded.Components.Job - if observed.isClusterUpdating() { + if shouldUpdateCluster(observed) { status.State = v1beta1.ClusterStateUpdating - } else if !isUpdateTriggered(&recorded.Revision) && isJobStopped(jobStatus) { + } else if !recorded.Revision.IsUpdateTriggered() && jobStatus.IsStopped() { var policy = observed.cluster.Spec.Job.CleanupPolicy if jobStatus.State == v1beta1.JobStateSucceeded && policy.AfterJobSucceeds != v1beta1.CleanupActionKeepCluster { @@ -430,8 +430,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( } case v1beta1.ClusterStateStopping, v1beta1.ClusterStatePartiallyStopped: - //if isClusterUpdating { - if observed.isClusterUpdating() { + if shouldUpdateCluster(observed) { status.State = v1beta1.ClusterStateUpdating } else if runningComponents == 0 { status.State = v1beta1.ClusterStateStopped @@ -441,7 +440,7 @@ func (updater *ClusterStatusUpdater) deriveClusterStatus( status.State = v1beta1.ClusterStateStopping } case v1beta1.ClusterStateStopped: - if isUpdateTriggered(&recorded.Revision) { + if recorded.Revision.IsUpdateTriggered() { status.State = v1beta1.ClusterStateUpdating } else { status.State = v1beta1.ClusterStateStopped @@ -535,13 +534,13 @@ func (updater *ClusterStatusUpdater) deriveJobStatus() *v1beta1.JobStatus { switch { case oldJob == nil: newJobState = v1beta1.JobStatePending - case observed.isClusterUpdating(): + case shouldUpdateJob(&observed): newJobState = v1beta1.JobStateUpdating - case shouldRestartJob(jobSpec, oldJob): + case oldJob.ShouldRestart(jobSpec): newJobState = v1beta1.JobStateRestarting - case isJobPending(oldJob) && oldJob.DeployTime != "": + case oldJob.IsPending() && oldJob.DeployTime != "": newJobState = v1beta1.JobStateDeploying - case isJobStopped(oldJob): + case oldJob.IsStopped(): newJobState = oldJob.State // Derive the job state from the observed Flink job, if it exists. case observedFlinkJob != nil: @@ -567,10 +566,13 @@ func (updater *ClusterStatusUpdater) deriveJobStatus() *v1beta1.JobStatus { var submitterState = observedSubmitter.getState() if submitterState == JobDeployStateUnknown { newJobState = v1beta1.JobStateLost - // Case in which the job submission clearly fails even if it is not confirmed by JobManager - // Job submitter is deployed but failed. - } else if submitterState == JobDeployStateFailed { + break + } + // Case in which the job submission clearly fails even if it is not confirmed by JobManager + // Job submitter is deployed but failed. + if submitterState == JobDeployStateFailed { newJobState = v1beta1.JobStateDeployFailed + break } newJobState = oldJob.State } @@ -583,17 +585,13 @@ func (updater *ClusterStatusUpdater) deriveJobStatus() *v1beta1.JobStatus { if oldJob == nil || oldJob.State != newJob.State { // TODO: It would be ideal to set the times with the timestamp retrieved from the Flink API like /jobs/{job-id}. switch { - case isJobPending(newJob): + case newJob.IsPending(): newJob.DeployTime = "" if newJob.State == v1beta1.JobStateUpdating { newJob.RestartCount = 0 } else if newJob.State == v1beta1.JobStateRestarting { newJob.RestartCount++ } - case newJob.State == v1beta1.JobStateDeploying: - newJob.ID = "" - newJob.StartTime = "" - newJob.EndTime = "" case newJob.State == v1beta1.JobStateRunning: setTimestamp(&newJob.StartTime) newJob.EndTime = "" @@ -601,7 +599,7 @@ func (updater *ClusterStatusUpdater) deriveJobStatus() *v1beta1.JobStatus { if oldJob.FinalSavepoint { newJob.FinalSavepoint = false } - case isJobStopped(newJob): + case newJob.IsStopped(): if newJob.EndTime == "" { setTimestamp(&newJob.EndTime) } @@ -826,7 +824,7 @@ func (updater *ClusterStatusUpdater) deriveSavepointStatus( // Check failure conditions of savepoint in progress. if s.State == v1beta1.SavepointStateInProgress { switch { - case isJobStopped(newJobStatus): + case newJobStatus.IsStopped(): errMsg = "Flink job is stopped." s.State = v1beta1.SavepointStateFailed case flinkJobID == nil: @@ -837,7 +835,8 @@ func (updater *ClusterStatusUpdater) deriveSavepointStatus( s.State = v1beta1.SavepointStateFailed } } - + // TODO: Record event or introduce Condition in CRD status to notify update state pended. + // https://github.com/kubernetes/apimachinery/blob/57f2a0733447cfd41294477d833cce6580faaca3/pkg/apis/meta/v1/types.go#L1376 // Make up message. if errMsg != "" { if s.TriggerReason == v1beta1.SavepointTriggerReasonUpdate { @@ -877,19 +876,17 @@ func deriveControlStatus( case v1beta1.ControlNameJobCancel: if newSavepoint.State == v1beta1.SavepointStateSucceeded && newJob.State == v1beta1.JobStateCancelled { c.State = v1beta1.ControlStateSucceeded - } else if isJobStopped(newJob) { + } else if newJob.IsStopped() { c.Message = "Aborted job cancellation: savepoint is not completed, but job is stopped already." c.State = v1beta1.ControlStateFailed - } else if newSavepoint.TriggerReason == v1beta1.SavepointTriggerReasonJobCancel && - (newSavepoint.State == v1beta1.SavepointStateFailed || newSavepoint.State == v1beta1.SavepointStateTriggerFailed) { + } else if newSavepoint.IsFailed() && newSavepoint.TriggerReason == v1beta1.SavepointTriggerReasonJobCancel { c.Message = "Aborted job cancellation: failed to take savepoint." c.State = v1beta1.ControlStateFailed } case v1beta1.ControlNameSavepoint: if newSavepoint.State == v1beta1.SavepointStateSucceeded { c.State = v1beta1.ControlStateSucceeded - } else if newSavepoint.TriggerReason == v1beta1.SavepointTriggerReasonUserRequested && - (newSavepoint.State == v1beta1.SavepointStateFailed || newSavepoint.State == v1beta1.SavepointStateTriggerFailed) { + } else if newSavepoint.IsFailed() && newSavepoint.TriggerReason == v1beta1.SavepointTriggerReasonUserRequested { c.State = v1beta1.ControlStateFailed } } diff --git a/controllers/flinkcluster_util.go b/controllers/flinkcluster_util.go index 8a9f7098..bb140084 100644 --- a/controllers/flinkcluster_util.go +++ b/controllers/flinkcluster_util.go @@ -20,7 +20,7 @@ import ( "bytes" "encoding/json" "fmt" - v1beta1 "github.com/googlecloudplatform/flink-operator/api/v1beta1" + "github.com/googlecloudplatform/flink-operator/api/v1beta1" "github.com/googlecloudplatform/flink-operator/controllers/flinkclient" "github.com/googlecloudplatform/flink-operator/controllers/history" appsv1 "k8s.io/api/apps/v1" @@ -42,7 +42,7 @@ const ( RevisionNameLabel = "flinkoperator.k8s.io/revision-name" - SavepointRequestRetryIntervalSec = 10 + SavepointRetryIntervalSeconds = 10 ) type UpdateState string @@ -137,17 +137,22 @@ func setTimestamp(target *string) { *target = tc.ToString(now) } +func getTime(timeStr string) time.Time { + var tc TimeConverter + return tc.FromString(timeStr) +} + func isBlank(s *string) bool { - return s == nil || *s == "" || strings.TrimSpace(*s) == "" + return s == nil || strings.TrimSpace(*s) == "" } // Checks whether it is possible to take savepoint. func canTakeSavepoint(cluster v1beta1.FlinkCluster) bool { var jobSpec = cluster.Spec.Job var savepointStatus = cluster.Status.Savepoint - var jobStatus = cluster.Status.Components.Job + var job = cluster.Status.Components.Job return jobSpec != nil && jobSpec.SavepointsDir != nil && - !isJobStopped(jobStatus) && + !job.IsStopped() && (savepointStatus == nil || savepointStatus.State != v1beta1.SavepointStateInProgress) } @@ -158,18 +163,6 @@ func shouldStopJob(cluster *v1beta1.FlinkCluster) bool { (cancelRequested != nil && *cancelRequested) } -// shouldRestartJob returns true if the controller should restart failed or lost job. -// The controller can restart the job only if there is a fresh savepoint to restore, recorded in status field. -func shouldRestartJob( - jobSpec *v1beta1.JobSpec, - jobStatus *v1beta1.JobStatus) bool { - var restartEnabled = jobSpec.RestartPolicy != nil && *jobSpec.RestartPolicy == v1beta1.JobRestartPolicyFromSavepointOnFailure - if restartEnabled && isJobFailed(jobStatus) && isSavepointUpToDate(jobSpec, jobStatus) { - return true - } - return false -} - func getFromSavepoint(jobSpec batchv1.JobSpec) string { var jobArgs = jobSpec.Template.Spec.Containers[0].Args for i, arg := range jobArgs { @@ -346,63 +339,11 @@ func getSavepointEvent(status v1beta1.SavepointStatus) (eventType string, eventR return } -func isJobActive(j *v1beta1.JobStatus) bool { - return j != nil && - (j.State == v1beta1.JobStateRunning || j.State == v1beta1.JobStateDeploying) -} - -func isJobPending(j *v1beta1.JobStatus) bool { - return j != nil && - (j.State == v1beta1.JobStatePending || - j.State == v1beta1.JobStateUpdating || - j.State == v1beta1.JobStateRestarting) -} - -func isJobFailed(j *v1beta1.JobStatus) bool { - return j != nil && - (j.State == v1beta1.JobStateFailed || - j.State == v1beta1.JobStateLost || - j.State == v1beta1.JobStateDeployFailed) -} - -func isJobStopped(j *v1beta1.JobStatus) bool { - return j != nil && - (j.State == v1beta1.JobStateSucceeded || - j.State == v1beta1.JobStateCancelled || - j.State == v1beta1.JobStateFailed || - j.State == v1beta1.JobStateLost || - j.State == v1beta1.JobStateDeployFailed) -} - -func isUpdateTriggered(r *v1beta1.RevisionStatus) bool { - return r.CurrentRevision != r.NextRevision -} - func isUserControlFinished(controlStatus *v1beta1.FlinkClusterControlStatus) bool { return controlStatus.State == v1beta1.ControlStateSucceeded || controlStatus.State == v1beta1.ControlStateFailed } -// Check if the savepoint is up to date. -func isSavepointUpToDate(jobSpec *v1beta1.JobSpec, jobStatus *v1beta1.JobStatus) bool { - if jobStatus.FinalSavepoint { - return true - } - if jobSpec.MaxStateAgeToRestoreSeconds == nil || - jobStatus.SavepointLocation == "" || - jobStatus.SavepointTime == "" || - jobStatus.EndTime == "" { - return false - } - var tc = &TimeConverter{} - var jobEndTime = tc.FromString(jobStatus.EndTime) - var stateMaxAge = int(*jobSpec.MaxStateAgeToRestoreSeconds) - if !hasTimeElapsed(jobStatus.SavepointTime, jobEndTime, stateMaxAge) { - return true - } - return false -} - // Check time has passed func hasTimeElapsed(timeToCheckStr string, now time.Time, intervalSec int) bool { tc := &TimeConverter{} @@ -419,7 +360,7 @@ func hasTimeElapsed(timeToCheckStr string, now time.Time, intervalSec int) bool // If the component is not observed and it is required, then it is not updated yet. // If the component is not observed and it is optional, but it is specified in the spec, then it is not updated yet. func isComponentUpdated(component runtime.Object, cluster *v1beta1.FlinkCluster) bool { - if !isUpdateTriggered(&cluster.Status.Revision) { + if !cluster.Status.Revision.IsUpdateTriggered() { return true } switch o := component.(type) { @@ -486,7 +427,7 @@ func isUpdatedAll(observed ObservedClusterState) bool { // isClusterUpdateToDate checks whether all cluster components are replaced to next revision. func isClusterUpdateToDate(observed *ObservedClusterState) bool { - if !isUpdateTriggered(&observed.cluster.Status.Revision) { + if !observed.cluster.Status.Revision.IsUpdateTriggered() { return true } components := []runtime.Object{ @@ -513,18 +454,6 @@ func finalSavepointRequested(jobID string, s *v1beta1.SavepointStatus) bool { s.TriggerReason == v1beta1.SavepointTriggerReasonJobCancel) } -func updateReady(jobSpec *v1beta1.JobSpec, job *v1beta1.JobStatus) bool { - var takeSavepoint bool - if jobSpec != nil { - takeSavepoint = jobSpec.TakeSavepointOnUpdate == nil || *jobSpec.TakeSavepointOnUpdate - } - return job == nil || - !isBlank(jobSpec.FromSavepoint) || - !takeSavepoint || - (isJobActive(job) && job.FinalSavepoint) || - (!isJobActive(job) && isSavepointUpToDate(jobSpec, job)) -} - func getUpdateState(observed *ObservedClusterState) UpdateState { if observed.cluster == nil { return "" @@ -534,11 +463,11 @@ func getUpdateState(observed *ObservedClusterState) UpdateState { var job = recorded.Components.Job var jobSpec = observed.cluster.Spec.Job - if !isUpdateTriggered(&revision) { + if !revision.IsUpdateTriggered() { return "" } switch { - case isJobActive(job) || !updateReady(jobSpec, job): + case !job.UpdateReady(jobSpec, observed.observeTime): return UpdateStatePreparing case !isClusterUpdateToDate(observed): return UpdateStateInProgress @@ -546,6 +475,15 @@ func getUpdateState(observed *ObservedClusterState) UpdateState { return UpdateStateFinished } +func shouldUpdateJob(observed *ObservedClusterState) bool { + return observed.updateState == UpdateStateInProgress +} + +func shouldUpdateCluster(observed *ObservedClusterState) bool { + var job = observed.cluster.Status.Components.Job + return !job.IsActive() && observed.updateState == UpdateStateInProgress +} + func getNonLiveHistory(revisions []*appsv1.ControllerRevision, historyLimit int) []*appsv1.ControllerRevision { history := append([]*appsv1.ControllerRevision{}, revisions...) diff --git a/controllers/flinkcluster_util_test.go b/controllers/flinkcluster_util_test.go index d0f9e6cf..26089648 100644 --- a/controllers/flinkcluster_util_test.go +++ b/controllers/flinkcluster_util_test.go @@ -17,6 +17,8 @@ limitations under the License. package controllers import ( + v1beta1 "github.com/googlecloudplatform/flink-operator/api/v1beta1" + "gotest.tools/assert" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -26,10 +28,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "os" "testing" - "time" - - v1beta1 "github.com/googlecloudplatform/flink-operator/api/v1beta1" - "gotest.tools/assert" ) func TestTimeConverter(t *testing.T) { @@ -46,71 +44,6 @@ func TestTimeConverter(t *testing.T) { assert.Assert(t, str3 == str4) } -func TestShouldRestartJob(t *testing.T) { - var tc = &TimeConverter{} - var restartOnFailure = v1beta1.JobRestartPolicyFromSavepointOnFailure - var neverRestart = v1beta1.JobRestartPolicyNever - var maxStateAgeToRestoreSeconds = int32(300) // 5 min - - // Restart with savepoint up to date - var savepointTime = tc.ToString(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) - var endTime = tc.ToString(time.Date(2020, 1, 1, 0, 1, 0, 0, time.UTC)) - var jobSpec = v1beta1.JobSpec{ - RestartPolicy: &restartOnFailure, - MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, - } - var jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - SavepointLocation: "gs://my-bucket/savepoint-123", - SavepointTime: savepointTime, - EndTime: endTime, - } - var restart = shouldRestartJob(&jobSpec, &jobStatus) - assert.Equal(t, restart, true) - - // Not restart without savepoint - jobSpec = v1beta1.JobSpec{ - RestartPolicy: &restartOnFailure, - MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, - } - jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - EndTime: endTime, - } - restart = shouldRestartJob(&jobSpec, &jobStatus) - assert.Equal(t, restart, false) - - // Not restart with restartPolicy Never - jobSpec = v1beta1.JobSpec{ - RestartPolicy: &neverRestart, - MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, - } - jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - SavepointLocation: "gs://my-bucket/savepoint-123", - SavepointTime: savepointTime, - EndTime: endTime, - } - restart = shouldRestartJob(&jobSpec, &jobStatus) - assert.Equal(t, restart, false) - - // Not restart with old savepoint - savepointTime = tc.ToString(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) - endTime = tc.ToString(time.Date(2020, 1, 1, 0, 5, 0, 0, time.UTC)) - jobSpec = v1beta1.JobSpec{ - RestartPolicy: &neverRestart, - MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, - } - jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - SavepointLocation: "gs://my-bucket/savepoint-123", - SavepointTime: savepointTime, - EndTime: endTime, - } - restart = shouldRestartJob(&jobSpec, &jobStatus) - assert.Equal(t, restart, false) -} - func TestGetRetryCount(t *testing.T) { var data1 = map[string]string{} var result1, _ = getRetryCount(data1) @@ -270,46 +203,6 @@ func TestGetNextRevisionNumber(t *testing.T) { assert.Equal(t, nextRevision, int64(3)) } -func TestIsSavepointUpToDate(t *testing.T) { - var tc = &TimeConverter{} - var savepointTime = time.Now() - var jobEndTime = savepointTime.Add(time.Second * 100) - var maxStateAgeToRestoreSeconds = int32(300) - var jobSpec = v1beta1.JobSpec{ - MaxStateAgeToRestoreSeconds: &maxStateAgeToRestoreSeconds, - } - var jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - SavepointTime: tc.ToString(savepointTime), - SavepointLocation: "gs://my-bucket/savepoint-123", - } - var update = isSavepointUpToDate(&jobSpec, &jobStatus) - assert.Equal(t, update, true) - - // old - savepointTime = time.Now() - jobEndTime = savepointTime.Add(time.Second * 500) - jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - SavepointTime: tc.ToString(savepointTime), - SavepointLocation: "gs://my-bucket/savepoint-123", - EndTime: tc.ToString(jobEndTime), - } - update = isSavepointUpToDate(&jobSpec, &jobStatus) - assert.Equal(t, update, false) - - // Fails without savepointLocation - savepointTime = time.Now() - jobEndTime = savepointTime.Add(time.Second * 500) - jobStatus = v1beta1.JobStatus{ - State: v1beta1.JobStateFailed, - SavepointTime: tc.ToString(savepointTime), - EndTime: tc.ToString(jobEndTime), - } - update = isSavepointUpToDate(&jobSpec, &jobStatus) - assert.Equal(t, update, false) -} - func TestIsComponentUpdated(t *testing.T) { var cluster = v1beta1.FlinkCluster{ Status: v1beta1.FlinkClusterStatus{Revision: v1beta1.RevisionStatus{NextRevision: "cluster-85dc8f749-2"}}, @@ -380,7 +273,9 @@ func TestGetUpdateState(t *testing.T) { JobManager: v1beta1.JobManagerSpec{Ingress: &v1beta1.JobManagerIngressSpec{}}, Job: &v1beta1.JobSpec{}, }, - Status: v1beta1.FlinkClusterStatus{Revision: v1beta1.RevisionStatus{CurrentRevision: "cluster-85dc8f749-2", NextRevision: "cluster-aa5e3a87z-3"}}, + Status: v1beta1.FlinkClusterStatus{ + Revision: v1beta1.RevisionStatus{CurrentRevision: "cluster-85dc8f749-2", NextRevision: "cluster-aa5e3a87z-3"}, + }, }, jmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-aa5e3a87z"}}}, tmStatefulSet: &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{RevisionNameLabel: "cluster-85dc8f749"}}},