[server] track more startWorkspace failures

fixes #12332

Author: svenefftinge

components/server/src/prometheus-metrics.ts

@@ -148,7 +148,8 @@ const instanceStartsFailedTotal = new prometheusClient.Counter({
     registers: [prometheusClient.register],
 });
 
-export function increaseFailedInstanceStartCounter(reason: "clusterSelectionFailed" | "startOnClusterFailed") {
+export type FailedInstanceStartReason = "clusterSelectionFailed" | "startOnClusterFailed" | "other";
+export function increaseFailedInstanceStartCounter(reason: FailedInstanceStartReason) {
     instanceStartsFailedTotal.inc({ reason });
 }
 

components/server/src/workspace/workspace-starter.ts

@@ -112,7 +112,11 @@ import { WithReferrerContext } from "@gitpod/gitpod-protocol/lib/protocol";
 import { IDEOption, IDEOptions } from "@gitpod/gitpod-protocol/lib/ide-protocol";
 import { Deferred } from "@gitpod/gitpod-protocol/lib/util/deferred";
 import { ExtendedUser } from "@gitpod/ws-manager/lib/constraints";
-import { increaseFailedInstanceStartCounter, increaseSuccessfulInstanceStartCounter } from "../prometheus-metrics";
+import {
+    FailedInstanceStartReason,
+    increaseFailedInstanceStartCounter,
+    increaseSuccessfulInstanceStartCounter,
+} from "../prometheus-metrics";
 import { ContextParser } from "./context-parser-service";
 import { IDEService } from "../ide-service";
 import { WorkspaceClusterImagebuilderClientProvider } from "./workspace-cluster-imagebuilder-client-provider";
@@ -244,6 +248,12 @@ export async function getWorkspaceClassForInstance(
     }
 }
 
+class StartInstanceError extends Error {
+    constructor(public readonly reason: FailedInstanceStartReason, public readonly cause: Error) {
+        super("Starting workspace instance failed: " + cause.message);
+    }
+}
+
 @injectable()
 export class WorkspaceStarter {
     @inject(WorkspaceManagerClientProvider) protected readonly clientProvider: WorkspaceManagerClientProvider;
@@ -414,6 +424,11 @@ export class WorkspaceStarter {
                 forceRebuild,
             );
         } catch (e) {
+            let failedReason: FailedInstanceStartReason = "other";
+            if (e instanceof StartInstanceError) {
+                failedReason = e.reason;
+            }
+            increaseFailedInstanceStartCounter(failedReason);
             TraceContext.setError({ span }, e);
             throw e;
         } finally {
@@ -523,16 +538,14 @@ export class WorkspaceStarter {
                     await new Promise((resolve) => setTimeout(resolve, INSTANCE_START_RETRY_INTERVAL_SECONDS * 1000));
                 }
             } catch (err) {
-                increaseFailedInstanceStartCounter("startOnClusterFailed");
                 await this.failInstanceStart({ span }, err, workspace, instance);
-                throw err;
+                throw new StartInstanceError("startOnClusterFailed", err);
             }
 
             if (!resp) {
-                increaseFailedInstanceStartCounter("clusterSelectionFailed");
                 const err = new Error("cannot start a workspace because no workspace clusters are available");
                 await this.failInstanceStart({ span }, err, workspace, instance);
-                throw err;
+                throw new StartInstanceError("clusterSelectionFailed", err);
             }
             increaseSuccessfulInstanceStartCounter(retries);
 
@@ -579,6 +592,11 @@ export class WorkspaceStarter {
                 throw err;
             } else {
                 log.error("error starting instance", err, { instanceId: instance.id });
+                let failedReason: FailedInstanceStartReason = "other";
+                if (err instanceof StartInstanceError) {
+                    failedReason = err.reason;
+                }
+                increaseFailedInstanceStartCounter(failedReason);
             }
 
             return { instanceID: instance.id };