ws-manager: Prevent to slip the CREATING phase

Author: utam0k

components/ws-manager/pkg/manager/monitor.go

@@ -317,14 +317,19 @@ func actOnPodEvent(ctx context.Context, m actingManager, manager *Manager, statu
 		// The workspace has been scheduled on the cluster which means that we can start initializing it
 		go func() {
 			err := m.initializeWorkspaceContent(ctx, pod)
-
 			if err != nil {
 				// workspace initialization failed, which means the workspace as a whole failed
 				err = m.markWorkspace(ctx, workspaceID, addMark(workspaceExplicitFailAnnotation, err.Error()))
 				if err != nil {
 					log.WithError(err).Warn("was unable to mark workspace as failed")
 				}
 			}
+
+			err = m.markWorkspace(ctx, workspaceID, addMark(alreadyInitializingAnnotation, util.BooleanTrueString))
+			if err != nil {
+				log.WithError(err).Warn("was unable to mark workspace as already initializing")
+			}
+
 		}()
 
 	case api.WorkspacePhase_INITIALIZING:

components/ws-manager/pkg/manager/pod_controller.go

@@ -33,36 +33,34 @@ type PodReconciler struct {
 // For more details, check Reconcile and its Result here:
 // - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
 func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	go func() {
-		var pod corev1.Pod
-		err := r.Client.Get(context.Background(), req.NamespacedName, &pod)
-		if errors.IsNotFound(err) {
-			// pod is gone - that's ok
-			if pod, ok := r.Pods[req.NamespacedName]; ok {
-				delete(r.Pods, req.NamespacedName)
-				queue := pod.Annotations[workspaceIDAnnotation]
-				if queue == "" {
-					return
-				}
-				r.Monitor.eventpool.Add(queue, watch.Event{
-					Type:   watch.Deleted,
-					Object: &pod,
-				})
+	var pod corev1.Pod
+	err := r.Client.Get(context.Background(), req.NamespacedName, &pod)
+	if errors.IsNotFound(err) {
+		// pod is gone - that's ok
+		if pod, ok := r.Pods[req.NamespacedName]; ok {
+			delete(r.Pods, req.NamespacedName)
+			queue := pod.Annotations[workspaceIDAnnotation]
+			if queue == "" {
+				return ctrl.Result{}, nil
 			}
-			return
+			r.Monitor.eventpool.Add(queue, watch.Event{
+				Type:   watch.Deleted,
+				Object: &pod,
+			})
 		}
-		r.Pods[req.NamespacedName] = pod
+		return ctrl.Result{}, nil
+	}
+	r.Pods[req.NamespacedName] = pod
 
-		queue := pod.Annotations[workspaceIDAnnotation]
-		if queue == "" {
-			return
-		}
+	queue := pod.Annotations[workspaceIDAnnotation]
+	if queue == "" {
+		return ctrl.Result{}, nil
+	}
 
-		r.Monitor.eventpool.Add(queue, watch.Event{
-			Type:   watch.Modified,
-			Object: &pod,
-		})
-	}()
+	r.Monitor.eventpool.Add(queue, watch.Event{
+		Type:   watch.Modified,
+		Object: &pod,
+	})
 
 	return ctrl.Result{}, nil
 }

components/ws-manager/pkg/manager/status.go

@@ -125,6 +125,22 @@ func (wso *workspaceObjects) WasEverReady() (res bool) {
 	return true
 }
 
+func (wso *workspaceObjects) AlreadyInitializing() (res bool) {
+	check := func(a map[string]string) bool {
+		v, ok := a[alreadyInitializingAnnotation]
+		if !ok {
+			return false
+		}
+		return v == util.BooleanTrueString
+	}
+
+	if wso.Pod != nil {
+		return check(wso.Pod.Annotations)
+	}
+
+	return true
+}
+
 // HostName returns the name of the node this workspace is/was deployed to. If this workspace has never been deployed anywhere, HostName returns an empty string.
 func (wso *workspaceObjects) NodeName() string {
 	if wso.Pod == nil {
@@ -486,6 +502,13 @@ func (m *Manager) extractStatusFromPod(result *api.WorkspaceStatus, wso workspac
 			return nil
 		}
 
+		// the pod phase of pending is somtimes slipped because it may not be PENDING at the stage of retrieving information on the workspace pod.
+		if !wso.AlreadyInitializing() {
+			result.Phase = api.WorkspacePhase_CREATING
+			result.Message = "pod is running but init has not started yet or is not finished"
+			return nil
+		}
+
 		if _, neverReady := pod.Annotations[workspaceNeverReadyAnnotation]; !neverReady {
 			// workspcae has been marked ready by a workspace-ready probe of the monitor
 			result.Phase = api.WorkspacePhase_RUNNING