[ws-manager] fix workspace status flipping pending to deleted

components/ws-manager/pkg/manager/annotations.go

@@ -77,6 +77,9 @@ const (
 
 	// stoppedByRequestAnnotation is set on a pod when it was requested to stop using a StopWorkspace call
 	stoppedByRequestAnnotation = "gitpod.io/stoppedByRequest"
+
+	// attemptingToCreatePodAnnotation is set when ws-manager is trying to create pod and is removed when pod is successfully scheduled on the node
+	attemptingToCreatePodAnnotation = "gitpod.io/attemptingToCreate"
 )
 
 // markWorkspaceAsReady adds annotations to a workspace pod

components/ws-manager/pkg/manager/manager.go

@@ -200,6 +200,9 @@ func (m *Manager) StartWorkspace(ctx context.Context, req *api.StartWorkspaceReq
 	}
 	span.LogKV("event", "pod description created")
 
+	// add an annotation to the pod to signal that ws-manager will now attempt to create this pod
+	pod.Annotations[attemptingToCreatePodAnnotation] = "true"
+
 	// create the Pod in the cluster and wait until is scheduled
 	// https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.22.md#workloads-that-saturate-nodes-with-pods-may-see-pods-that-fail-due-to-node-admission
 	backoff := wait.Backoff{
@@ -290,6 +293,13 @@ func (m *Manager) StartWorkspace(ctx context.Context, req *api.StartWorkspaceReq
 		return nil, xerrors.Errorf("cannot create workspace pod: %w", err)
 	}
 
+	// remove annotation to signal that workspace pod was indeed created and scheduled on the node
+	err = m.markWorkspace(ctx, req.Id, deleteMark(attemptingToCreatePodAnnotation))
+	if err != nil {
+		clog.WithError(err).WithField("pod.Namespace", pod.Namespace).WithField("pod.Name", pod.Name).Error("failed to remove annotation after creating workspace pod. this will break things")
+		return nil, xerrors.Errorf("couldn't remove annotation after creating workspace pod: %w", err)
+	}
+
 	span.LogKV("event", "pod started successfully")
 
 	// all workspaces get a service now

components/ws-manager/pkg/manager/status.go

@@ -326,6 +326,13 @@ func (m *Manager) extractStatusFromPod(result *api.WorkspaceStatus, wso workspac
 		}
 	}
 
+	// if ws-manager is still attempting to create workspace pod, keep status in pending
+	// pod might get deleted several times if we cannot schedule it on the node
+	if _, atc := pod.Annotations[attemptingToCreatePodAnnotation]; atc {
+		result.Phase = api.WorkspacePhase_PENDING
+		return nil
+	}
+
 	if isPodBeingDeleted(pod) {
 		result.Phase = api.WorkspacePhase_STOPPING