diff --git a/chart/templates/ws-manager-configmap.yaml b/chart/templates/ws-manager-configmap.yaml index ebe4d63685ab2e..ed6c074dda502b 100644 --- a/chart/templates/ws-manager-configmap.yaml +++ b/chart/templates/ws-manager-configmap.yaml @@ -82,6 +82,7 @@ data: "headlessWorkspace": "60m", "initialization": "30m", "regularWorkspace": "30m", + "maxLifetime": "36h", "startup": "60m", "contentFinalization": "60m", "stopping": "60m", diff --git a/components/installer/pkg/components/ws-manager/configmap.go b/components/installer/pkg/components/ws-manager/configmap.go index 592237325f129d..1ea2bd61c8b892 100644 --- a/components/installer/pkg/components/ws-manager/configmap.go +++ b/components/installer/pkg/components/ws-manager/configmap.go @@ -86,6 +86,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { HeadlessWorkspace: util.Duration(1 * time.Hour), Initialization: util.Duration(30 * time.Minute), RegularWorkspace: util.Duration(30 * time.Minute), + MaxLifetime: util.Duration(36 * time.Hour), TotalStartup: util.Duration(1 * time.Hour), ContentFinalization: util.Duration(1 * time.Hour), Stopping: util.Duration(1 * time.Hour), diff --git a/components/ws-manager-api/go/config/config.go b/components/ws-manager-api/go/config/config.go index b6f1abdacb794a..6886f54fc176b8 100644 --- a/components/ws-manager-api/go/config/config.go +++ b/components/ws-manager-api/go/config/config.go @@ -122,6 +122,8 @@ type WorkspaceTimeoutConfiguration struct { Initialization util.Duration `json:"initialization"` // RegularWorkspace is the time a regular workspace can be without activity before it's shutdown RegularWorkspace util.Duration `json:"regularWorkspace"` + // MaxLifetime is the maximum lifetime of a regular workspace + MaxLifetime util.Duration `json:"maxLifetime"` // HeadlessWorkspace is the maximum runtime a headless workspace can have (including startup) HeadlessWorkspace util.Duration `json:"headlessWorkspace"` // AfterClose is the time a workspace lives after it has been marked closed @@ -188,6 +190,7 @@ func (c *Configuration) Validate() error { validation.Field(&c.Timeouts.HeadlessWorkspace, validation.Required), validation.Field(&c.Timeouts.Initialization, validation.Required), validation.Field(&c.Timeouts.RegularWorkspace, validation.Required), + validation.Field(&c.Timeouts.MaxLifetime, validation.Required), validation.Field(&c.Timeouts.TotalStartup, validation.Required), validation.Field(&c.Timeouts.ContentFinalization, validation.Required), validation.Field(&c.Timeouts.Stopping, validation.Required), diff --git a/components/ws-manager/config-schema.json b/components/ws-manager/config-schema.json index 63e163d64a46e4..9c47e7d81e90ac 100644 --- a/components/ws-manager/config-schema.json +++ b/components/ws-manager/config-schema.json @@ -173,6 +173,7 @@ "startup", "initialization", "regularWorkspace", + "maxLifetime", "headlessWorkspace", "afterClose", "stopping", @@ -194,6 +195,9 @@ "regularWorkspace": { "type": "string" }, + "maxLifetime": { + "type": "string" + }, "startup": { "type": "string" }, diff --git a/components/ws-manager/example-config.json b/components/ws-manager/example-config.json index a618f1d7884b8f..15d0688efd0883 100644 --- a/components/ws-manager/example-config.json +++ b/components/ws-manager/example-config.json @@ -41,6 +41,7 @@ "headlessWorkspace": "60m", "initialization": "30m", "regularWorkspace": "30m", + "maxLifetime": "36h", "startup": "30m" }, "eventTraceLog": "-", diff --git a/components/ws-manager/pkg/manager/integration_test.go b/components/ws-manager/pkg/manager/integration_test.go index ce98a226b798ed..45b614363bfdc5 100644 --- a/components/ws-manager/pkg/manager/integration_test.go +++ b/components/ws-manager/pkg/manager/integration_test.go @@ -87,6 +87,7 @@ func forIntegrationTestGetManager(t *testing.T) *Manager { Initialization: util.Duration(30 * time.Minute), TotalStartup: util.Duration(45 * time.Minute), RegularWorkspace: util.Duration(60 * time.Minute), + MaxLifetime: util.Duration(36 * time.Hour), HeadlessWorkspace: util.Duration(90 * time.Minute), Stopping: util.Duration(60 * time.Minute), ContentFinalization: util.Duration(15 * time.Minute), diff --git a/components/ws-manager/pkg/manager/monitor.go b/components/ws-manager/pkg/manager/monitor.go index bba13b9d8fa158..25af3e5cf1ec03 100644 --- a/components/ws-manager/pkg/manager/monitor.go +++ b/components/ws-manager/pkg/manager/monitor.go @@ -976,7 +976,7 @@ func (m *Monitor) finalizeWorkspaceContent(ctx context.Context, wso *workspaceOb } } -// markTimedoutWorkspaces finds workspaces which haven't been active recently and marks them as timed out +// markTimedoutWorkspaces finds workspaces which can be timeout due to inactivity or max lifetime allowed func (m *Monitor) markTimedoutWorkspaces(ctx context.Context) (err error) { span, ctx := tracing.FromContext(ctx, "markTimedoutWorkspaces") defer tracing.FinishSpan(span, &err) diff --git a/components/ws-manager/pkg/manager/status.go b/components/ws-manager/pkg/manager/status.go index 048f514b2e4998..8d2c5b354c257c 100644 --- a/components/ws-manager/pkg/manager/status.go +++ b/components/ws-manager/pkg/manager/status.go @@ -667,6 +667,7 @@ const ( activityPullingImages activity = "pulling images" activityRunningHeadless activity = "running the headless workspace" activityNone activity = "period of inactivity" + activityMaxLifetime activity = "maximum lifetime" activityClosed activity = "after being closed" activityInterrupted activity = "workspace interruption" activityStopping activity = "stopping" @@ -716,6 +717,11 @@ func (m *Manager) isWorkspaceTimedOut(wso workspaceObjects) (reason string, err return decide(start, m.Config.Timeouts.TotalStartup, activity) case api.WorkspacePhase_RUNNING: + // First check is always for the max lifetime + if msg, err := decide(start, m.Config.Timeouts.MaxLifetime, activityMaxLifetime); msg != "" { + return msg, err + } + timeout := m.Config.Timeouts.RegularWorkspace activity := activityNone if wso.IsWorkspaceHeadless() { diff --git a/components/ws-manager/pkg/manager/status_test.go b/components/ws-manager/pkg/manager/status_test.go index b89c36791638d9..eee333210b43c1 100644 --- a/components/ws-manager/pkg/manager/status_test.go +++ b/components/ws-manager/pkg/manager/status_test.go @@ -48,6 +48,7 @@ func TestIsWorkspaceTimedout(t *testing.T) { Initialization: util.Duration(30 * time.Minute), TotalStartup: util.Duration(45 * time.Minute), RegularWorkspace: util.Duration(60 * time.Minute), + MaxLifetime: util.Duration(36 * time.Hour), HeadlessWorkspace: util.Duration(90 * time.Minute), Stopping: util.Duration(60 * time.Minute), ContentFinalization: util.Duration(55 * time.Minute), diff --git a/components/ws-manager/pkg/manager/testdata/timeout_regular_maxLifetime.golden b/components/ws-manager/pkg/manager/testdata/timeout_regular_maxLifetime.golden new file mode 100644 index 00000000000000..1acf2c46bcfc2c --- /dev/null +++ b/components/ws-manager/pkg/manager/testdata/timeout_regular_maxLifetime.golden @@ -0,0 +1,3 @@ +{ + "reason": "workspace timed out after maximum lifetime (36h10m) took longer than 36h00m" +} \ No newline at end of file diff --git a/components/ws-manager/pkg/manager/testdata/timeout_regular_maxLifetime.json b/components/ws-manager/pkg/manager/testdata/timeout_regular_maxLifetime.json new file mode 100644 index 00000000000000..69cb786d88934c --- /dev/null +++ b/components/ws-manager/pkg/manager/testdata/timeout_regular_maxLifetime.json @@ -0,0 +1,291 @@ +{ + "creationDelta": "2170m", + "activity": "1m", + "wso": { + "pod": { + "metadata": { + "name": "ws-foobas", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/ws-foobas", + "uid": "486e5f88-4354-11e9-aee4-080027861af1", + "resourceVersion": "64956", + "creationTimestamp": "2019-03-10T16:48:08Z", + "labels": { + "gpwsman": "true", + "headless": "false", + "owner": "foobar", + "metaID": "metameta", + "workspaceID": "foobas", + "workspaceType": "regular" + }, + "annotations": { + "gitpod/id": "foobas", + "gitpod/ready": "true", + "gitpod/servicePrefix": "foobas", + "gitpod/url": "http://10.0.0.114:8082", + "prometheus.io/path": "/metrics", + "prometheus.io/port": "23000", + "prometheus.io/scrape": "true" + } + }, + "spec": { + "volumes": [ + { + "name": "vol-this-workspace", + "hostPath": { + "path": "/tmp/workspaces/foobas", + "type": "DirectoryOrCreate" + } + }, + { + "name": "vol-sync-tmp", + "hostPath": { + "path": "/tmp/workspaces/sync-tmp", + "type": "DirectoryOrCreate" + } + }, + { + "name": "default-token-6qnvx", + "secret": { + "secretName": "default-token-6qnvx", + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "workspace", + "image": "nginx:latest", + "ports": [ + { + "containerPort": 23000, + "protocol": "TCP" + } + ], + "env": [ + { + "name": "THEIA_WORKSPACE_ROOT", + "value": "/workspace" + }, + { + "name": "GITPOD_THEIA_PORT", + "value": "23000" + }, + { + "name": "GITPOD_HOST", + "value": "gitpod.io" + }, + { + "name": "GITPOD_INTERVAL", + "value": "30" + }, + { + "name": "GITPOD_WSSYNC_APITOKEN", + "value": "c17a7eaf-e5de-4e9d-815a-7919379e2bf8" + }, + { + "name": "GITPOD_WSSYNC_APIPORT", + "value": "44444" + }, + { + "name": "GITPOD_REPO_ROOT", + "value": "/workspace" + }, + { + "name": "GITPOD_CLI_APITOKEN", + "value": "690516e2-c416-4a28-ba74-e36f125922aa" + }, + { + "name": "GITPOD_WORKSPACE_ID", + "value": "foobas" + }, + { + "name": "GITPOD_GIT_USER_NAME", + "value": "usernameGoesHere" + }, + { + "name": "GITPOD_GIT_USER_EMAIL", + "value": "some@user.com" + } + ], + "resources": { + "limits": { + "cpu": "100m", + "memory": "100Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + }, + "volumeMounts": [ + { + "name": "vol-this-workspace", + "mountPath": "/workspace" + }, + { + "name": "default-token-6qnvx", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "Always" + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "serviceAccountName": "default", + "serviceAccount": "default", + "nodeName": "minikube", + "securityContext": {}, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + } + ] + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2019-03-10T16:48:08Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2019-03-10T16:48:13Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2019-03-10T16:48:08Z" + } + ], + "hostIP": "10.0.2.15", + "podIP": "172.17.0.5", + "startTime": "2019-03-10T16:48:08Z", + "containerStatuses": [ + { + "name": "sync", + "state": { + "running": { + "startedAt": "2019-03-10T16:48:13Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "csweichel/noop:latest", + "imageID": "docker-pullable://csweichel/noop@sha256:aaa6b993f4c853fac7101aa7fc087926f829004e62cbce6e1852e5a3aac87c52", + "containerID": "docker://9961f75ea72f36bb0ba1e42b3b2da98eb44a9dc12e7c7e8edfb52512b3b04016" + }, + { + "name": "workspace", + "state": { + "running": { + "startedAt": "2019-03-10T16:48:12Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "nginx:latest", + "imageID": "docker-pullable://nginx@sha256:98efe605f61725fd817ea69521b0eeb32bef007af0e3d0aeb6258c6e6fe7fc1a", + "containerID": "docker://e7080b843a47db414d6c94cfda7f657b99d8aa5bbf7c9c118ec98c0eefb6c0df" + } + ], + "qosClass": "Guaranteed" + } + }, + "theiaService": { + "metadata": { + "name": "foobas-theia", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/services/foobas-theia", + "uid": "48687212-4354-11e9-aee4-080027861af1", + "resourceVersion": "64923", + "creationTimestamp": "2019-03-10T16:48:08Z", + "labels": { + "gpwsman": "true", + "headless": "false", + "owner": "foobar", + "metaID": "metameta", + "workspaceID": "foobas" + } + }, + "spec": { + "ports": [ + { + "name": "theia", + "protocol": "TCP", + "port": 23000, + "targetPort": 23000 + } + ], + "selector": { + "gpwsman": "true", + "headless": "false", + "owner": "foobar", + "workspaceID": "foobas" + }, + "clusterIP": "10.103.194.121", + "type": "ClusterIP", + "sessionAffinity": "None" + }, + "status": { + "loadBalancer": {} + } + }, + "portsService": { + "metadata": { + "name": "foobas-ports", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/services/foobas-ports", + "uid": "486cb304-4354-11e9-aee4-080027861af1", + "resourceVersion": "64926", + "creationTimestamp": "2019-03-10T16:48:08Z", + "labels": { + "gpwsman": "true", + "workspaceID": "foobas" + } + }, + "spec": { + "ports": [ + { + "protocol": "TCP", + "port": 8080, + "targetPort": 8080 + } + ], + "selector": { + "gpwsman": "true", + "workspaceID": "foobas" + }, + "clusterIP": "10.110.184.222", + "type": "ClusterIP", + "sessionAffinity": "None" + }, + "status": { + "loadBalancer": {} + } + } + } +} \ No newline at end of file diff --git a/components/ws-manager/pkg/manager/testing_test.go b/components/ws-manager/pkg/manager/testing_test.go index 6e0bd8569a1f8d..7a98ff3bd37d0f 100644 --- a/components/ws-manager/pkg/manager/testing_test.go +++ b/components/ws-manager/pkg/manager/testing_test.go @@ -56,6 +56,7 @@ func forTestingOnlyManagerConfig() config.Configuration { Initialization: util.Duration(30 * time.Minute), TotalStartup: util.Duration(45 * time.Minute), RegularWorkspace: util.Duration(60 * time.Minute), + MaxLifetime: util.Duration(36 * time.Hour), HeadlessWorkspace: util.Duration(90 * time.Minute), Stopping: util.Duration(60 * time.Minute), ContentFinalization: util.Duration(55 * time.Minute),