[ws-manager] fix workspace status flipping pending to deleted

gitpod-io · Apr 21, 2022 · 652967e · 652967e
1 parent 7e1b76f
commit 652967e
Show file tree

Hide file tree

Showing 26 changed files with 40 additions and 0 deletions.
diff --git a/components/ws-manager/pkg/manager/annotations.go b/components/ws-manager/pkg/manager/annotations.go
@@ -77,6 +77,9 @@ const (
 
 	// stoppedByRequestAnnotation is set on a pod when it was requested to stop using a StopWorkspace call
 	stoppedByRequestAnnotation = "gitpod.io/stoppedByRequest"
+
+	// attemptingToCreatePodAnnotation is set when ws-manager is trying to create pod and is removed when pod is successfully scheduled on the node
+	attemptingToCreatePodAnnotation = "gitpod.io/attemptingToCreate"
 )
 
 // markWorkspaceAsReady adds annotations to a workspace pod

diff --git a/components/ws-manager/pkg/manager/create.go b/components/ws-manager/pkg/manager/create.go
@@ -306,6 +306,7 @@ func (m *Manager) createDefiniteWorkspacePod(startContext *startWorkspaceContext
 		kubernetes.WorkspaceImageSpecAnnotation: imageSpec,
 		kubernetes.OwnerTokenAnnotation:         startContext.OwnerToken,
 		wsk8s.TraceIDAnnotation:                 startContext.TraceID,
+		attemptingToCreatePodAnnotation:         "true",
 		// TODO(cw): post Kubernetes 1.19 use GA form for settings those profiles
 		"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
 		// We're using a custom seccomp profile for user namespaces to allow clone, mount and chroot.

diff --git a/components/ws-manager/pkg/manager/manager.go b/components/ws-manager/pkg/manager/manager.go
@@ -290,6 +290,13 @@ func (m *Manager) StartWorkspace(ctx context.Context, req *api.StartWorkspaceReq
 		return nil, xerrors.Errorf("cannot create workspace pod: %w", err)
 	}
 
+	// remove annotation to signal that workspace pod was indeed created and scheduled on the node
+	err = m.markWorkspace(ctx, req.Id, deleteMark(attemptingToCreatePodAnnotation))
+	if err != nil {
+		clog.WithError(err).WithField("pod.Namespace", pod.Namespace).WithField("pod.Name", pod.Name).Error("failed to remove annotation after creating workspace pod. this will break things")
+		return nil, xerrors.Errorf("couldn't remove annotation after creating workspace pod: %w", err)
+	}
+
 	span.LogKV("event", "pod started successfully")
 
 	// all workspaces get a service now

diff --git a/components/ws-manager/pkg/manager/status.go b/components/ws-manager/pkg/manager/status.go
@@ -326,6 +326,13 @@ func (m *Manager) extractStatusFromPod(result *api.WorkspaceStatus, wso workspac
 		}
 	}
 
+	// if ws-manager is still attempting to create workspace pod, keep status in pending
+	// pod might get deleted several times if we cannot schedule it on the node
+	if _, atc := pod.Annotations[attemptingToCreatePodAnnotation]; atc {
+		result.Phase = api.WorkspacePhase_PENDING
+		return nil
+	}
+
 	if isPodBeingDeleted(pod) {
 		result.Phase = api.WorkspacePhase_STOPPING
 

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_admission.golden b/components/ws-manager/pkg/manager/testdata/cdwp_admission.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_everyone",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_affinity.golden b/components/ws-manager/pkg/manager/testdata/cdwp_affinity.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_customcerts.golden b/components/ws-manager/pkg/manager/testdata/cdwp_customcerts.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_empty_resource_req.golden b/components/ws-manager/pkg/manager/testdata/cdwp_empty_resource_req.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_envvars.golden b/components/ws-manager/pkg/manager/testdata/cdwp_envvars.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_fixedresources.golden b/components/ws-manager/pkg/manager/testdata/cdwp_fixedresources.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod.io/cpuLimit": "900m",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_fullworkspacebackup.golden b/components/ws-manager/pkg/manager/testdata/cdwp_fullworkspacebackup.golden
@@ -19,6 +19,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/fullWorkspaceBackup": "true",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_imagebuild.golden b/components/ws-manager/pkg/manager/testdata/cdwp_imagebuild.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "foobar",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_imagebuild_template.golden b/components/ws-manager/pkg/manager/testdata/cdwp_imagebuild_template.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "foobar",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_no_ideimage.golden b/components/ws-manager/pkg/manager/testdata/cdwp_no_ideimage.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_prebuild.golden b/components/ws-manager/pkg/manager/testdata/cdwp_prebuild.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "foobar",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_prebuild_template.golden b/components/ws-manager/pkg/manager/testdata/cdwp_prebuild_template.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "foobar",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_prebuild_template_override_resources.golden b/components/ws-manager/pkg/manager/testdata/cdwp_prebuild_template_override_resources.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "foobar",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_probe.golden b/components/ws-manager/pkg/manager/testdata/cdwp_probe.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "foobar",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_readinessprobe.golden b/components/ws-manager/pkg/manager/testdata/cdwp_readinessprobe.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_tasks.golden b/components/ws-manager/pkg/manager/testdata/cdwp_tasks.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_template.golden b/components/ws-manager/pkg/manager/testdata/cdwp_template.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_timeout.golden b/components/ws-manager/pkg/manager/testdata/cdwp_timeout.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/customTimeout": "35m20s",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_userns.golden b/components/ws-manager/pkg/manager/testdata/cdwp_userns.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_with_ephemeral_storage.golden b/components/ws-manager/pkg/manager/testdata/cdwp_with_ephemeral_storage.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "foobar",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_withaffinity_regular.golden b/components/ws-manager/pkg/manager/testdata/cdwp_withaffinity_regular.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "test",

diff --git a/components/ws-manager/pkg/manager/testdata/cdwp_withaffinityheadless.golden b/components/ws-manager/pkg/manager/testdata/cdwp_withaffinityheadless.golden
@@ -18,6 +18,7 @@
             "annotations": {
                 "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
                 "container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
+                "gitpod.io/attemptingToCreate": "true",
                 "gitpod/admission": "admit_owner_only",
                 "gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
                 "gitpod/id": "foobar",