From 062a78c2ccec6c2fd69b57b9696183ff5f0523e3 Mon Sep 17 00:00:00 2001 From: "Laurie T. Malau" Date: Mon, 26 Sep 2022 15:47:43 +0000 Subject: [PATCH] [bridge] Mark as stopped pending and stopping --- components/ws-manager-bridge/src/bridge.ts | 61 ++++++++++--------- components/ws-manager-bridge/src/config.ts | 2 + .../testdata/render/aws-setup/output.golden | 6 +- .../testdata/render/azure-setup/output.golden | 6 +- .../render/customization/output.golden | 6 +- .../render/external-registry/output.golden | 6 +- .../testdata/render/gcp-setup/output.golden | 6 +- .../testdata/render/http-proxy/output.golden | 6 +- .../render/insecure-s3-setup/output.golden | 6 +- .../cmd/testdata/render/minimal/output.golden | 6 +- .../statefulset-customization/output.golden | 6 +- .../use-pod-security-policies/output.golden | 6 +- .../workspace-requests-limits/output.golden | 6 +- .../components/ws-manager-bridge/configmap.go | 2 + .../pkg/components/ws-manager-bridge/types.go | 2 + 15 files changed, 83 insertions(+), 50 deletions(-) diff --git a/components/ws-manager-bridge/src/bridge.ts b/components/ws-manager-bridge/src/bridge.ts index 623f769e1618f9..43c97ddcd43b54 100644 --- a/components/ws-manager-bridge/src/bridge.ts +++ b/components/ws-manager-bridge/src/bridge.ts @@ -452,7 +452,13 @@ export class WorkspaceManagerBridge implements Disposable { // Control running workspace instances against ws-manager try { - await this.controlNonStoppedWSManagerManagedInstances(ctx, nonStoppedInstances, clientProvider); + await this.controlNonStoppedWSManagerManagedInstances( + ctx, + nonStoppedInstances, + clientProvider, + this.config.timeouts.pendingPhaseSeconds, + this.config.timeouts.stoppingPhaseSeconds, + ); disconnectStarted = Number.MAX_SAFE_INTEGER; // Reset disconnect period } catch (err) { @@ -489,6 +495,8 @@ export class WorkspaceManagerBridge implements Disposable { parentCtx: TraceContext, runningInstances: RunningWorkspaceInfo[], clientProvider: ClientProvider, + pendingPhaseSeconds: number, + stoppingPhaseSeconds: number, ) { const installation = this.config.installation; @@ -508,36 +516,33 @@ export class WorkspaceManagerBridge implements Disposable { for (const [instanceId, ri] of runningInstancesIdx.entries()) { const instance = ri.latestInstance; const phase = instance.status.phase; - if (phase !== "running") { - // This below if block is to validate the planned fix - if ( - phase === "pending" || - phase === "creating" || - phase === "initializing" || - (phase === "stopping" && - instance.stoppingTime && - durationLongerThanSeconds(Date.parse(instance.stoppingTime), 10)) - ) { - log.info( - { instanceId, workspaceId: instance.workspaceId }, - "Logging to validate #12902. Should mark as stopped in database.", - { installation }, - { phase }, - ); - } - log.debug({ instanceId }, "Skipping instance", { - phase: instance.status.phase, - creationTime: instance.creationTime, - region: instance.region, - }); + + // When ws-manager is not aware of the following instances outside of the timeout duration, + // they should be marked as stopped. + // pending states timeout is 1 hour after creationTime. + // stopping states timeout is 1 hour after stoppingTime. + if ( + phase === "running" || + (phase === "pending" && + durationLongerThanSeconds(Date.parse(instance.creationTime), pendingPhaseSeconds)) || + (phase === "stopping" && + instance.stoppingTime && + durationLongerThanSeconds(Date.parse(instance.stoppingTime), stoppingPhaseSeconds)) + ) { + log.info( + { instanceId, workspaceId: instance.workspaceId }, + "Database says the instance is present, but ws-man does not know about it. Marking as stopped in database.", + { installation, phase }, + ); + await this.markWorkspaceInstanceAsStopped(ctx, ri, new Date()); continue; } - log.info( - "Database says the instance is running, but wsman does not know about it. Marking as stopped in database.", - { instanceId, workspaceId: instance.workspaceId, installation, phase }, - ); - await this.markWorkspaceInstanceAsStopped(ctx, ri, new Date()); + log.debug({ instanceId }, "Skipping instance", { + phase: phase, + creationTime: instance.creationTime, + region: instance.region, + }); } log.debug("Done controlling running instances.", { installation }); diff --git a/components/ws-manager-bridge/src/config.ts b/components/ws-manager-bridge/src/config.ts index 2ab8adc2d12e78..af6c178e89d592 100644 --- a/components/ws-manager-bridge/src/config.ts +++ b/components/ws-manager-bridge/src/config.ts @@ -31,6 +31,8 @@ export interface Configuration { preparingPhaseSeconds: number; buildingPhaseSeconds: number; unknownPhaseSeconds: number; + pendingPhaseSeconds: number; + stoppingPhaseSeconds: number; }; // emulatePreparingIntervalSeconds configures how often we check for Workspaces in phase "preparing" for clusters we do not govern diff --git a/install/installer/cmd/testdata/render/aws-setup/output.golden b/install/installer/cmd/testdata/render/aws-setup/output.golden index 26d1121f44cf7c..805701b4a62b30 100644 --- a/install/installer/cmd/testdata/render/aws-setup/output.golden +++ b/install/installer/cmd/testdata/render/aws-setup/output.golden @@ -4855,7 +4855,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -8929,7 +8931,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f5d95af2308e35194d44403c42e269fbe9e4c596d138796fa95f705ec3b4d352 + gitpod.io/checksum_config: eb3706cacfc9d2ff32216e0f3564bf211676885e1511d13147c812cb94000ee0 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/azure-setup/output.golden b/install/installer/cmd/testdata/render/azure-setup/output.golden index 58cbb6f65bee6b..6a4ffc454aa404 100644 --- a/install/installer/cmd/testdata/render/azure-setup/output.golden +++ b/install/installer/cmd/testdata/render/azure-setup/output.golden @@ -4714,7 +4714,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -8774,7 +8776,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f5d95af2308e35194d44403c42e269fbe9e4c596d138796fa95f705ec3b4d352 + gitpod.io/checksum_config: eb3706cacfc9d2ff32216e0f3564bf211676885e1511d13147c812cb94000ee0 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/customization/output.golden b/install/installer/cmd/testdata/render/customization/output.golden index 37a856b2d9ec4a..a7b88c38844ab1 100644 --- a/install/installer/cmd/testdata/render/customization/output.golden +++ b/install/installer/cmd/testdata/render/customization/output.golden @@ -5681,7 +5681,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -10360,7 +10362,7 @@ spec: metadata: annotations: gitpod.io: hello - gitpod.io/checksum_config: fdb0c0adb6ea187bd587fb7c286084b4853439fdfafa7f32bbdcacee744e5c3e + gitpod.io/checksum_config: 2d645aacfa07b22cc2e8f2a80d3b797031f470a1af284ab923fc9678ff81efac hello: world creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/external-registry/output.golden b/install/installer/cmd/testdata/render/external-registry/output.golden index a31659066d98e1..42628801661442 100644 --- a/install/installer/cmd/testdata/render/external-registry/output.golden +++ b/install/installer/cmd/testdata/render/external-registry/output.golden @@ -4901,7 +4901,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -9200,7 +9202,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 124c0235366403cca5d28e6b8c8cb08d44991225ff7bba386100ac942b78dd48 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/gcp-setup/output.golden b/install/installer/cmd/testdata/render/gcp-setup/output.golden index 262cf15df8b9ce..06e5fa6b495fa2 100644 --- a/install/installer/cmd/testdata/render/gcp-setup/output.golden +++ b/install/installer/cmd/testdata/render/gcp-setup/output.golden @@ -4673,7 +4673,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -8697,7 +8699,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 6d44bef12145215794685e52c3139f04195bcf8e7ca1f1d7b3803da845e837eb + gitpod.io/checksum_config: 6901b3da5bcf091d43e67f04b108c86334788b912c8c27c18afd0f0025dcd3de creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/http-proxy/output.golden b/install/installer/cmd/testdata/render/http-proxy/output.golden index 330bd49d6378ba..b08115def35259 100644 --- a/install/installer/cmd/testdata/render/http-proxy/output.golden +++ b/install/installer/cmd/testdata/render/http-proxy/output.golden @@ -5124,7 +5124,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -10863,7 +10865,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 124c0235366403cca5d28e6b8c8cb08d44991225ff7bba386100ac942b78dd48 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/insecure-s3-setup/output.golden b/install/installer/cmd/testdata/render/insecure-s3-setup/output.golden index a5a01abd07db52..d7bbb116a128b1 100644 --- a/install/installer/cmd/testdata/render/insecure-s3-setup/output.golden +++ b/install/installer/cmd/testdata/render/insecure-s3-setup/output.golden @@ -5037,7 +5037,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -9352,7 +9354,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 124c0235366403cca5d28e6b8c8cb08d44991225ff7bba386100ac942b78dd48 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/minimal/output.golden b/install/installer/cmd/testdata/render/minimal/output.golden index f114a5b244d9e2..96814d30199a34 100644 --- a/install/installer/cmd/testdata/render/minimal/output.golden +++ b/install/installer/cmd/testdata/render/minimal/output.golden @@ -5121,7 +5121,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -9575,7 +9577,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 124c0235366403cca5d28e6b8c8cb08d44991225ff7bba386100ac942b78dd48 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/statefulset-customization/output.golden b/install/installer/cmd/testdata/render/statefulset-customization/output.golden index a08a3ad29c6c4e..f49f6d309aa3a2 100644 --- a/install/installer/cmd/testdata/render/statefulset-customization/output.golden +++ b/install/installer/cmd/testdata/render/statefulset-customization/output.golden @@ -5133,7 +5133,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -9587,7 +9589,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 124c0235366403cca5d28e6b8c8cb08d44991225ff7bba386100ac942b78dd48 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden b/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden index 60bb995a0cdb36..2a42225306c885 100644 --- a/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden +++ b/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden @@ -5454,7 +5454,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -10019,7 +10021,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 124c0235366403cca5d28e6b8c8cb08d44991225ff7bba386100ac942b78dd48 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden b/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden index 2fdff4543a9eaa..53e2855b53056f 100644 --- a/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden +++ b/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden @@ -5124,7 +5124,9 @@ data: "timeouts": { "preparingPhaseSeconds": 3600, "buildingPhaseSeconds": 3600, - "unknownPhaseSeconds": 600 + "unknownPhaseSeconds": 600, + "pendingPhaseSeconds": 3600, + "stoppingPhaseSeconds": 3600 }, "clusterSyncIntervalSeconds": 60 } @@ -9578,7 +9580,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: bacace964780fd56bdd04b997df0017c24563c2aac6bffd10962c297b54c91ba + gitpod.io/checksum_config: 124c0235366403cca5d28e6b8c8cb08d44991225ff7bba386100ac942b78dd48 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/pkg/components/ws-manager-bridge/configmap.go b/install/installer/pkg/components/ws-manager-bridge/configmap.go index 4963c3f3ad5b87..25b53d958d9c4d 100644 --- a/install/installer/pkg/components/ws-manager-bridge/configmap.go +++ b/install/installer/pkg/components/ws-manager-bridge/configmap.go @@ -28,6 +28,8 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { PreparingPhaseSeconds: 3600, BuildingPhaseSeconds: 3600, UnknownPhaseSeconds: 600, + PendingPhaseSeconds: 3600, + StoppingPhaseSeconds: 3600, }, EmulatePreparingIntervalSeconds: 10, StaticBridges: WSManagerList(ctx), diff --git a/install/installer/pkg/components/ws-manager-bridge/types.go b/install/installer/pkg/components/ws-manager-bridge/types.go index 2f49bc4a6b785b..6d23ce54abac99 100644 --- a/install/installer/pkg/components/ws-manager-bridge/types.go +++ b/install/installer/pkg/components/ws-manager-bridge/types.go @@ -26,6 +26,8 @@ type Timeouts struct { PreparingPhaseSeconds int32 `json:"preparingPhaseSeconds"` BuildingPhaseSeconds int32 `json:"buildingPhaseSeconds"` UnknownPhaseSeconds int32 `json:"unknownPhaseSeconds"` + PendingPhaseSeconds int32 `json:"pendingPhaseSeconds"` + StoppingPhaseSeconds int32 `json:"stoppingPhaseSeconds"` } // WorkspaceCluster from components/gitpod-protocol/src/workspace-cluster.ts