[bridge] Mark as stopped pending and stopping

Author: Laurie T. Malau

components/ws-manager-bridge/src/bridge.ts

@@ -508,36 +508,39 @@ export class WorkspaceManagerBridge implements Disposable {
             for (const [instanceId, ri] of runningInstancesIdx.entries()) {
                 const instance = ri.latestInstance;
                 const phase = instance.status.phase;
-                if (phase !== "running") {
-                    // This below if block is to validate the planned fix
-                    if (
-                        phase === "pending" ||
-                        phase === "creating" ||
-                        phase === "initializing" ||
-                        (phase === "stopping" &&
-                            instance.stoppingTime &&
-                            durationLongerThanSeconds(Date.parse(instance.stoppingTime), 10))
-                    ) {
-                        log.info(
-                            { instanceId, workspaceId: instance.workspaceId },
-                            "Logging to validate #12902. Should mark as stopped in database.",
-                            { installation },
-                            { phase },
-                        );
-                    }
-                    log.debug({ instanceId }, "Skipping instance", {
-                        phase: instance.status.phase,
-                        creationTime: instance.creationTime,
-                        region: instance.region,
-                    });
+
+                // When ws-manager is not aware of the following instances outside of the timeout duration,
+                // they should be marked as stopped.
+                // pending states timeout is 1 hour after creationTime.
+                // stopping states timeout is 1 hour after stoppingTime.
+                if (
+                    (phase === "pending" &&
+                        durationLongerThanSeconds(
+                            Date.parse(instance.creationTime),
+                            this.config.timeouts.maxInstanceDuration,
+                        )) ||
+                    (phase === "stopping" &&
+                        instance.stoppingTime &&
+                        durationLongerThanSeconds(
+                            Date.parse(instance.stoppingTime),
+                            this.config.timeouts.maxInstanceDuration,
+                        ))
+                ) {
+                    log.info(
+                        { instanceId, workspaceId: instance.workspaceId },
+                        "Database says the instance is running, but ws-man does not know about it. Marking as stopped in database.",
+                        { installation },
+                        { phase },
+                    );
+                    await this.markWorkspaceInstanceAsStopped(ctx, ri, new Date());
                     continue;
                 }
 
-                log.info(
-                    "Database says the instance is running, but wsman does not know about it. Marking as stopped in database.",
-                    { instanceId, workspaceId: instance.workspaceId, installation, phase },
-                );
-                await this.markWorkspaceInstanceAsStopped(ctx, ri, new Date());
+                log.debug({ instanceId }, "Skipping instance", {
+                    phase: phase,
+                    creationTime: instance.creationTime,
+                    region: instance.region,
+                });
             }
 
             log.debug("Done controlling running instances.", { installation });

components/ws-manager-bridge/src/config.ts

@@ -31,6 +31,7 @@ export interface Configuration {
         preparingPhaseSeconds: number;
         buildingPhaseSeconds: number;
         unknownPhaseSeconds: number;
+        maxInstanceDuration: number;
     };
 
     // emulatePreparingIntervalSeconds configures how often we check for Workspaces in phase "preparing" for clusters we do not govern

install/installer/cmd/testdata/render/aws-setup/output.golden

@@ -26,6 +26,7 @@ type Timeouts struct {
 	PreparingPhaseSeconds int32 `json:"preparingPhaseSeconds"`
 	BuildingPhaseSeconds  int32 `json:"buildingPhaseSeconds"`
 	UnknownPhaseSeconds   int32 `json:"unknownPhaseSeconds"`
+	MaxInstanceDuration   int32 `json:"maxInstanceDuration"`
 }
 
 // WorkspaceCluster from components/gitpod-protocol/src/workspace-cluster.ts