Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ws-manager] fix workspace status flipping pending to deleted #9438

Merged
merged 1 commit into from
Apr 21, 2022
Merged

[ws-manager] fix workspace status flipping pending to deleted #9438

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions components/ws-manager/pkg/manager/annotations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ const (

// stoppedByRequestAnnotation is set on a pod when it was requested to stop using a StopWorkspace call
stoppedByRequestAnnotation = "gitpod.io/stoppedByRequest"

// attemptingToCreatePodAnnotation is set when ws-manager is trying to create pod and is removed when pod is successfully scheduled on the node
attemptingToCreatePodAnnotation = "gitpod.io/attemptingToCreate"
)

// markWorkspaceAsReady adds annotations to a workspace pod
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,7 @@ func (m *Manager) createDefiniteWorkspacePod(startContext *startWorkspaceContext
kubernetes.WorkspaceImageSpecAnnotation: imageSpec,
kubernetes.OwnerTokenAnnotation: startContext.OwnerToken,
wsk8s.TraceIDAnnotation: startContext.TraceID,
attemptingToCreatePodAnnotation: "true",
// TODO(cw): post Kubernetes 1.19 use GA form for settings those profiles
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
// We're using a custom seccomp profile for user namespaces to allow clone, mount and chroot.
Expand Down
7 changes: 7 additions & 0 deletions components/ws-manager/pkg/manager/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,13 @@ func (m *Manager) StartWorkspace(ctx context.Context, req *api.StartWorkspaceReq
return nil, xerrors.Errorf("cannot create workspace pod: %w", err)
}

// remove annotation to signal that workspace pod was indeed created and scheduled on the node
err = m.markWorkspace(ctx, req.Id, deleteMark(attemptingToCreatePodAnnotation))
if err != nil {
clog.WithError(err).WithField("pod.Namespace", pod.Namespace).WithField("pod.Name", pod.Name).Error("failed to remove annotation after creating workspace pod. this will break things")
return nil, xerrors.Errorf("couldn't remove annotation after creating workspace pod: %w", err)
}

span.LogKV("event", "pod started successfully")

// all workspaces get a service now
Expand Down
7 changes: 7 additions & 0 deletions components/ws-manager/pkg/manager/status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,13 @@ func (m *Manager) extractStatusFromPod(result *api.WorkspaceStatus, wso workspac
}
}

// if ws-manager is still attempting to create workspace pod, keep status in pending
// pod might get deleted several times if we cannot schedule it on the node
if _, atc := pod.Annotations[attemptingToCreatePodAnnotation]; atc {
result.Phase = api.WorkspacePhase_PENDING
return nil
}

if isPodBeingDeleted(pod) {
result.Phase = api.WorkspacePhase_STOPPING

Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_admission.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_everyone",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_affinity.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_customcerts.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_empty_resource_req.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_envvars.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_fixedresources.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod.io/cpuLimit": "900m",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_fullworkspacebackup.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/fullWorkspaceBackup": "true",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_imagebuild.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "foobar",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_imagebuild_template.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "foobar",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_no_ideimage.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_prebuild.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "foobar",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_prebuild_template.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "foobar",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_prebuild_template_override_resources.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "foobar",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_probe.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "foobar",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_readinessprobe.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_tasks.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_template.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_timeout.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/customTimeout": "35m20s",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_userns.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_with_ephemeral_storage.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "foobar",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_withaffinity_regular.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "test",
Expand Down
1 change: 1 addition & 0 deletions components/ws-manager/pkg/manager/testdata/cdwp_withaffinityheadless.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"annotations": {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
"gitpod.io/attemptingToCreate": "true",
"gitpod/admission": "admit_owner_only",
"gitpod/contentInitializer": "GmcKZXdvcmtzcGFjZXMvY3J5cHRpYy1pZC1nb2VzLWhlcmcvZmQ2MjgwNGItNGNhYi0xMWU5LTg0M2EtNGU2NDUzNzMwNDhlLnRhckBnaXRwb2QtZGV2LXVzZXItY2hyaXN0ZXN0aW5n",
"gitpod/id": "foobar",
Expand Down