diff --git a/components/ws-manager-bridge/src/bridge.ts b/components/ws-manager-bridge/src/bridge.ts index 45272429b7ac3e..eedb8a28183dd0 100644 --- a/components/ws-manager-bridge/src/bridge.ts +++ b/components/ws-manager-bridge/src/bridge.ts @@ -452,7 +452,12 @@ export class WorkspaceManagerBridge implements Disposable { // Control running workspace instances against ws-manager try { - await this.controlRunningInstances(ctx, runningInstances, clientProvider); + await this.controlRunningInstances( + ctx, + runningInstances, + clientProvider, + this.config.maxTimeToRunningPhaseSeconds, + ); disconnectStarted = Number.MAX_SAFE_INTEGER; // Reset disconnect period } catch (err) { @@ -489,6 +494,7 @@ export class WorkspaceManagerBridge implements Disposable { parentCtx: TraceContext, runningInstances: RunningWorkspaceInfo[], clientProvider: ClientProvider, + maxTimeToRunningPhaseSeconds: number, ) { const installation = this.config.installation; @@ -506,7 +512,12 @@ export class WorkspaceManagerBridge implements Disposable { for (const [instanceId, ri] of runningInstancesIdx.entries()) { const instance = ri.latestInstance; - if (instance.status.phase !== "running") { + if ( + !( + instance.status.phase === "running" || + durationLongerThanSeconds(Date.parse(instance.creationTime), maxTimeToRunningPhaseSeconds) + ) + ) { log.debug({ instanceId }, "Skipping instance", { phase: instance.status.phase, creationTime: instance.creationTime, @@ -601,11 +612,10 @@ export class WorkspaceManagerBridge implements Disposable { protected async markWorkspaceInstanceAsStopped(ctx: TraceContext, info: RunningWorkspaceInfo, now: Date) { const nowISO = now.toISOString(); + info.latestInstance.status.phase = "stopped"; info.latestInstance.stoppingTime = nowISO; info.latestInstance.stoppedTime = nowISO; - info.latestInstance.status.phase = "stopped"; await this.workspaceDB.trace(ctx).storeInstance(info.latestInstance); - await this.messagebus.notifyOnInstanceUpdate(ctx, info.workspace.ownerId, info.latestInstance); await this.prebuildUpdater.stopPrebuildInstance(ctx, info.latestInstance); } diff --git a/components/ws-manager-bridge/src/config.ts b/components/ws-manager-bridge/src/config.ts index 2ab8adc2d12e78..d5d4b94c796838 100644 --- a/components/ws-manager-bridge/src/config.ts +++ b/components/ws-manager-bridge/src/config.ts @@ -38,4 +38,7 @@ export interface Configuration { // clusterSyncIntervalSeconds configures how often we sync workspace cluster information clusterSyncIntervalSeconds: number; + + // maxTimeToRunningPhaseSeconds is the timeout duration + maxTimeToRunningPhaseSeconds: number; } diff --git a/install/installer/cmd/testdata/render/aws-setup/output.golden b/install/installer/cmd/testdata/render/aws-setup/output.golden index 9b9604c7a4da40..57c2f5d3457c90 100644 --- a/install/installer/cmd/testdata/render/aws-setup/output.golden +++ b/install/installer/cmd/testdata/render/aws-setup/output.golden @@ -4576,7 +4576,8 @@ data: "buildingPhaseSeconds": 3600, "unknownPhaseSeconds": 600 }, - "clusterSyncIntervalSeconds": 60 + "clusterSyncIntervalSeconds": 60, + "maxTimeToRunningPhaseSeconds": 3600 } kind: ConfigMap metadata: @@ -8439,7 +8440,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f5d95af2308e35194d44403c42e269fbe9e4c596d138796fa95f705ec3b4d352 + gitpod.io/checksum_config: 846469058fd082ae195479935c67054553bb448d05cf029a19af738503147aa7 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/azure-setup/output.golden b/install/installer/cmd/testdata/render/azure-setup/output.golden index d675d310abde0c..4223c31e13b148 100644 --- a/install/installer/cmd/testdata/render/azure-setup/output.golden +++ b/install/installer/cmd/testdata/render/azure-setup/output.golden @@ -4436,7 +4436,8 @@ data: "buildingPhaseSeconds": 3600, "unknownPhaseSeconds": 600 }, - "clusterSyncIntervalSeconds": 60 + "clusterSyncIntervalSeconds": 60, + "maxTimeToRunningPhaseSeconds": 3600 } kind: ConfigMap metadata: @@ -8285,7 +8286,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f5d95af2308e35194d44403c42e269fbe9e4c596d138796fa95f705ec3b4d352 + gitpod.io/checksum_config: 846469058fd082ae195479935c67054553bb448d05cf029a19af738503147aa7 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/customization/output.golden b/install/installer/cmd/testdata/render/customization/output.golden index 19e216749c51ea..7cffa5ac58e490 100644 --- a/install/installer/cmd/testdata/render/customization/output.golden +++ b/install/installer/cmd/testdata/render/customization/output.golden @@ -5363,7 +5363,8 @@ data: "buildingPhaseSeconds": 3600, "unknownPhaseSeconds": 600 }, - "clusterSyncIntervalSeconds": 60 + "clusterSyncIntervalSeconds": 60, + "maxTimeToRunningPhaseSeconds": 3600 } kind: ConfigMap metadata: @@ -9817,7 +9818,7 @@ spec: metadata: annotations: gitpod.io: hello - gitpod.io/checksum_config: fdb0c0adb6ea187bd587fb7c286084b4853439fdfafa7f32bbdcacee744e5c3e + gitpod.io/checksum_config: 1af8949463afb559e837a140ecdb4ca73db58e3da8fb0680570a1cdada3c3abe hello: world creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/external-registry/output.golden b/install/installer/cmd/testdata/render/external-registry/output.golden index c3a2a236df9de3..90ad615ec80eac 100644 --- a/install/installer/cmd/testdata/render/external-registry/output.golden +++ b/install/installer/cmd/testdata/render/external-registry/output.golden @@ -4623,7 +4623,8 @@ data: "buildingPhaseSeconds": 3600, "unknownPhaseSeconds": 600 }, - "clusterSyncIntervalSeconds": 60 + "clusterSyncIntervalSeconds": 60, + "maxTimeToRunningPhaseSeconds": 3600 } kind: ConfigMap metadata: @@ -8711,7 +8712,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 19a15d16ea83f7cd1c61948f0e433f3f72ce175524ea639c1c050a485f17f74d creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/gcp-setup/output.golden b/install/installer/cmd/testdata/render/gcp-setup/output.golden index 5d3cac0a3f1eff..f8bd2d46fd7834 100644 --- a/install/installer/cmd/testdata/render/gcp-setup/output.golden +++ b/install/installer/cmd/testdata/render/gcp-setup/output.golden @@ -4395,7 +4395,8 @@ data: "buildingPhaseSeconds": 3600, "unknownPhaseSeconds": 600 }, - "clusterSyncIntervalSeconds": 60 + "clusterSyncIntervalSeconds": 60, + "maxTimeToRunningPhaseSeconds": 3600 } kind: ConfigMap metadata: @@ -8208,7 +8209,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 6d44bef12145215794685e52c3139f04195bcf8e7ca1f1d7b3803da845e837eb + gitpod.io/checksum_config: f0b5a025d81fce48adb9fd1ad8445422fa80b136acba6b2eb69263df3388c388 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/minimal/output.golden b/install/installer/cmd/testdata/render/minimal/output.golden index b8afc4b0a79cbd..d1d9e664adbe12 100644 --- a/install/installer/cmd/testdata/render/minimal/output.golden +++ b/install/installer/cmd/testdata/render/minimal/output.golden @@ -4843,7 +4843,8 @@ data: "buildingPhaseSeconds": 3600, "unknownPhaseSeconds": 600 }, - "clusterSyncIntervalSeconds": 60 + "clusterSyncIntervalSeconds": 60, + "maxTimeToRunningPhaseSeconds": 3600 } kind: ConfigMap metadata: @@ -9086,7 +9087,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 19a15d16ea83f7cd1c61948f0e433f3f72ce175524ea639c1c050a485f17f74d creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/statefulset-customization/output.golden b/install/installer/cmd/testdata/render/statefulset-customization/output.golden index 01759bcec44d55..0e7b671700b0d7 100644 --- a/install/installer/cmd/testdata/render/statefulset-customization/output.golden +++ b/install/installer/cmd/testdata/render/statefulset-customization/output.golden @@ -4855,7 +4855,8 @@ data: "buildingPhaseSeconds": 3600, "unknownPhaseSeconds": 600 }, - "clusterSyncIntervalSeconds": 60 + "clusterSyncIntervalSeconds": 60, + "maxTimeToRunningPhaseSeconds": 3600 } kind: ConfigMap metadata: @@ -9098,7 +9099,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 19a15d16ea83f7cd1c61948f0e433f3f72ce175524ea639c1c050a485f17f74d creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden b/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden index 315db49d03ece8..a7e31b28b766f0 100644 --- a/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden +++ b/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden @@ -5176,7 +5176,8 @@ data: "buildingPhaseSeconds": 3600, "unknownPhaseSeconds": 600 }, - "clusterSyncIntervalSeconds": 60 + "clusterSyncIntervalSeconds": 60, + "maxTimeToRunningPhaseSeconds": 3600 } kind: ConfigMap metadata: @@ -9530,7 +9531,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 19a15d16ea83f7cd1c61948f0e433f3f72ce175524ea639c1c050a485f17f74d creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden b/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden index ec08dbee28c45d..a474a92e62cdfe 100644 --- a/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden +++ b/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden @@ -4846,7 +4846,8 @@ data: "buildingPhaseSeconds": 3600, "unknownPhaseSeconds": 600 }, - "clusterSyncIntervalSeconds": 60 + "clusterSyncIntervalSeconds": 60, + "maxTimeToRunningPhaseSeconds": 3600 } kind: ConfigMap metadata: @@ -9089,7 +9090,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 19a15d16ea83f7cd1c61948f0e433f3f72ce175524ea639c1c050a485f17f74d creationTimestamp: null labels: app: gitpod diff --git a/install/installer/pkg/components/ws-manager-bridge/configmap.go b/install/installer/pkg/components/ws-manager-bridge/configmap.go index 4963c3f3ad5b87..65aee5e3834b2a 100644 --- a/install/installer/pkg/components/ws-manager-bridge/configmap.go +++ b/install/installer/pkg/components/ws-manager-bridge/configmap.go @@ -32,6 +32,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { EmulatePreparingIntervalSeconds: 10, StaticBridges: WSManagerList(ctx), ClusterSyncIntervalSeconds: 60, + MaxTimeToRunningPhaseSeconds: 60 * 60, } fc, err := common.ToJSONString(wsmbcfg) diff --git a/install/installer/pkg/components/ws-manager-bridge/types.go b/install/installer/pkg/components/ws-manager-bridge/types.go index 2f49bc4a6b785b..1fbe0dcf3e8c5c 100644 --- a/install/installer/pkg/components/ws-manager-bridge/types.go +++ b/install/installer/pkg/components/ws-manager-bridge/types.go @@ -15,6 +15,7 @@ type Configuration struct { EmulatePreparingIntervalSeconds int32 `json:"emulatePreparingIntervalSeconds"` Timeouts Timeouts `json:"timeouts"` ClusterSyncIntervalSeconds int32 `json:"clusterSyncIntervalSeconds"` + MaxTimeToRunningPhaseSeconds int32 `json:"maxTimeToRunningPhaseSeconds"` } type ClusterService struct {