diff --git a/components/ws-manager/pkg/manager/manager.go b/components/ws-manager/pkg/manager/manager.go index fe0fa3d27f0ae0..9c9cb9c00e7e44 100644 --- a/components/ws-manager/pkg/manager/manager.go +++ b/components/ws-manager/pkg/manager/manager.go @@ -235,22 +235,41 @@ func (m *Manager) StartWorkspace(ctx context.Context, req *api.StartWorkspaceReq retryErr = err var tempPod corev1.Pod - getErr := m.Clientset.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, &tempPod) - if getErr != nil { - clog.WithError(getErr).WithField("pod.Namespace", pod.Namespace).WithField("pod.Name", pod.Name).Error("was unable to get pod") - // pod doesn't exist, so we are safe to proceed with retry - return false, nil + finalizerRemoved := false + // We do below in loop as sometimes the pod might be already deleted or modified when we attempt to remove the finalizer + // so we first get the pod and then attempt to remove the finalizer + // in case the pod is not found in the first place, we return success + for i := 0; i < 3 && !finalizerRemoved; i++ { + getErr := m.Clientset.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, &tempPod) + if getErr != nil { + if k8serr.IsNotFound(getErr) { + // pod doesn't exist, so we are safe to proceed with retry + return false, nil + } + clog.WithError(getErr).WithField("pod.Namespace", pod.Namespace).WithField("pod.Name", pod.Name).Error("was unable to get pod") + // pod get call failed so we error out + return false, retryErr + } + // we successfully got the pod, now we attempt to remove finalizer + tempPod.Finalizers = []string{} + updateErr := m.Clientset.Update(ctx, &tempPod) + if updateErr != nil { + if k8serr.IsNotFound(updateErr) { + // pod doesn't exist, so we are safe to proceed with retry + return false, nil + } + clog.WithError(updateErr).WithField("pod.Namespace", pod.Namespace).WithField("pod.Name", pod.Name).Error("was unable to remove finalizer") + continue + } + finalizerRemoved = true } - tempPod.Finalizers = []string{} - updateErr := m.Clientset.Update(ctx, &tempPod) - if updateErr != nil { - clog.WithError(updateErr).WithField("pod.Namespace", pod.Namespace).WithField("pod.Name", pod.Name).Error("was unable to remove finalizer") - // failed to remove finalizer, we not going to be able to create a new pod, so bail out with retry error + // if even after retries we could not remove finalizers, then error out + if !finalizerRemoved { return false, retryErr } deleteErr := m.Clientset.Delete(ctx, &tempPod) - if deleteErr != nil { + if deleteErr != nil && !k8serr.IsNotFound(deleteErr) { clog.WithError(deleteErr).WithField("pod.Namespace", pod.Namespace).WithField("pod.Name", pod.Name).Error("was unable to delete pod") // failed to delete pod, so not going to be able to create a new pod, so bail out return false, retryErr