[bridge] Mark as stopped pending and stopping

Author: Laurie T. Malau

components/ws-manager-bridge/src/bridge.ts

@@ -452,7 +452,12 @@ export class WorkspaceManagerBridge implements Disposable {
 
                     // Control running workspace instances against ws-manager
                     try {
-                        await this.controlNonStoppedWSManagerManagedInstances(ctx, nonStoppedInstances, clientProvider);
+                        await this.controlNonStoppedWSManagerManagedInstances(
+                            ctx,
+                            nonStoppedInstances,
+                            clientProvider,
+                            this.config.timeouts.maxInstanceDuration,
+                        );
 
                         disconnectStarted = Number.MAX_SAFE_INTEGER; // Reset disconnect period
                     } catch (err) {
@@ -489,6 +494,7 @@ export class WorkspaceManagerBridge implements Disposable {
         parentCtx: TraceContext,
         runningInstances: RunningWorkspaceInfo[],
         clientProvider: ClientProvider,
+        maxInstanceDuration: number,
     ) {
         const installation = this.config.installation;
 
@@ -508,36 +514,33 @@ export class WorkspaceManagerBridge implements Disposable {
             for (const [instanceId, ri] of runningInstancesIdx.entries()) {
                 const instance = ri.latestInstance;
                 const phase = instance.status.phase;
-                if (phase !== "running") {
-                    // This below if block is to validate the planned fix
-                    if (
-                        phase === "pending" ||
-                        phase === "creating" ||
-                        phase === "initializing" ||
-                        (phase === "stopping" &&
-                            instance.stoppingTime &&
-                            durationLongerThanSeconds(Date.parse(instance.stoppingTime), 10))
-                    ) {
-                        log.info(
-                            { instanceId, workspaceId: instance.workspaceId },
-                            "Logging to validate #12902. Should mark as stopped in database.",
-                            { installation },
-                            { phase },
-                        );
-                    }
-                    log.debug({ instanceId }, "Skipping instance", {
-                        phase: instance.status.phase,
-                        creationTime: instance.creationTime,
-                        region: instance.region,
-                    });
+
+                // When ws-manager is not aware of the following instances outside of the timeout duration,
+                // they should be marked as stopped.
+                // pending states timeout is 1 hour after creationTime.
+                // stopping states timeout is 1 hour after stoppingTime.
+                if (
+                    (phase === "pending" &&
+                        durationLongerThanSeconds(Date.parse(instance.creationTime), maxInstanceDuration)) ||
+                    (phase === "stopping" &&
+                        instance.stoppingTime &&
+                        durationLongerThanSeconds(Date.parse(instance.stoppingTime), maxInstanceDuration))
+                ) {
+                    log.info(
+                        { instanceId, workspaceId: instance.workspaceId },
+                        "Database says the instance is running, but ws-man does not know about it. Marking as stopped in database.",
+                        { installation },
+                        { phase },
+                    );
+                    await this.markWorkspaceInstanceAsStopped(ctx, ri, new Date());
                     continue;
                 }
 
-                log.info(
-                    "Database says the instance is running, but wsman does not know about it. Marking as stopped in database.",
-                    { instanceId, workspaceId: instance.workspaceId, installation, phase },
-                );
-                await this.markWorkspaceInstanceAsStopped(ctx, ri, new Date());
+                log.debug({ instanceId }, "Skipping instance", {
+                    phase: phase,
+                    creationTime: instance.creationTime,
+                    region: instance.region,
+                });
             }
 
             log.debug("Done controlling running instances.", { installation });

components/ws-manager-bridge/src/config.ts

@@ -31,6 +31,7 @@ export interface Configuration {
         preparingPhaseSeconds: number;
         buildingPhaseSeconds: number;
         unknownPhaseSeconds: number;
+        maxInstanceDuration: number;
     };
 
     // emulatePreparingIntervalSeconds configures how often we check for Workspaces in phase "preparing" for clusters we do not govern

install/installer/cmd/testdata/render/aws-setup/output.golden

@@ -28,6 +28,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 			PreparingPhaseSeconds: 3600,
 			BuildingPhaseSeconds:  3600,
 			UnknownPhaseSeconds:   600,
+			MaxInstanceDuration:   3600,
 		},
 		EmulatePreparingIntervalSeconds: 10,
 		StaticBridges:                   WSManagerList(ctx),

install/installer/cmd/testdata/render/azure-setup/output.golden

@@ -26,6 +26,7 @@ type Timeouts struct {
 	PreparingPhaseSeconds int32 `json:"preparingPhaseSeconds"`
 	BuildingPhaseSeconds  int32 `json:"buildingPhaseSeconds"`
 	UnknownPhaseSeconds   int32 `json:"unknownPhaseSeconds"`
+	MaxInstanceDuration   int32 `json:"maxInstanceDuration"`
 }
 
 // WorkspaceCluster from components/gitpod-protocol/src/workspace-cluster.ts