[server] Add a liveness probe for event loop lag

components/server/src/config.ts

@@ -173,6 +173,12 @@ export interface ConfigSerialized {
 
     vsxRegistryUrl: string;
 
+    /*
+     * The maximum event loop lag allowed before the liveness endpoint should return
+     * an error code.
+     */
+    maximumEventLoopLag: number;
+
     /**
      * Payment related options
      */

components/server/src/container-module.ts

@@ -110,6 +110,7 @@ import {
 } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
 import { VerificationService } from "./auth/verification-service";
 import { WebhookEventGarbageCollector } from "./projects/webhook-event-garbage-collector";
+import { LivenessController } from "./liveness/liveness-controller";
 
 export const productionContainerModule = new ContainerModule((bind, unbind, isBound, rebind) => {
     bind(Config).toConstantValue(ConfigFile.fromFile());
@@ -218,6 +219,7 @@ export const productionContainerModule = new ContainerModule((bind, unbind, isBo
 
     bind(WorkspaceGarbageCollector).toSelf().inSingletonScope();
     bind(WorkspaceDownloadService).toSelf().inSingletonScope();
+    bind(LivenessController).toSelf().inSingletonScope();
 
     bind(OneTimeSecretServer).toSelf().inSingletonScope();
 

components/server/src/liveness/liveness-controller.ts

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import { injectable, inject } from "inversify";
+import * as express from "express";
+import * as prom from "prom-client";
+import { Config } from "../config";
+import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
+
+@injectable()
+export class LivenessController {
+    @inject(Config) protected readonly config: Config;
+
+    get apiRouter(): express.Router {
+        const router = express.Router();
+        this.addLivenessHandler(router);
+        return router;
+    }
+
+    protected addLivenessHandler(router: express.Router) {
+        router.get("/", async (_, res) => {
+            if (!this.config.maximumEventLoopLag) {
+                res.end();
+                return;
+            }
+            const metrics = await prom.register.getMetricsAsJSON();
+            const metric = metrics.find((m) => m.name === "nodejs_eventloop_lag_seconds");
+            if (!metric) {
+                log.error("unable to find nodejs event loop metric, liveness probe will not function");
+                res.end();
+                return;
+            }
+            const value = (metric as any).values[0].value;
+            res.status(value > this.config.maximumEventLoopLag ? 500 : 200);
+            res.end(`${value}`);
+        });
+    }
+}

components/server/src/server.ts

@@ -49,6 +49,7 @@ import { LocalMessageBroker } from "./messaging/local-message-broker";
 import { WsConnectionHandler } from "./express/ws-connection-handler";
 import { InstallationAdminController } from "./installation-admin/installation-admin-controller";
 import { WebhookEventGarbageCollector } from "./projects/webhook-event-garbage-collector";
+import { LivenessController } from "./liveness/liveness-controller";
 
 @injectable()
 export class Server<C extends GitpodClient, S extends GitpodServer> {
@@ -65,6 +66,7 @@ export class Server<C extends GitpodClient, S extends GitpodServer> {
     @inject(MessageBusIntegration) protected readonly messagebus: MessageBusIntegration;
     @inject(LocalMessageBroker) protected readonly localMessageBroker: LocalMessageBroker;
     @inject(WorkspaceDownloadService) protected readonly workspaceDownloadService: WorkspaceDownloadService;
+    @inject(LivenessController) protected readonly livenessController: LivenessController;
     @inject(MonitoringEndpointsApp) protected readonly monitoringEndpointsApp: MonitoringEndpointsApp;
     @inject(CodeSyncService) private readonly codeSyncService: CodeSyncService;
     @inject(HeadlessLogController) protected readonly headlessLogController: HeadlessLogController;
@@ -298,6 +300,7 @@ export class Server<C extends GitpodClient, S extends GitpodServer> {
         app.use("/code-sync", this.codeSyncService.apiRouter);
         app.use(HEADLESS_LOGS_PATH_PREFIX, this.headlessLogController.headlessLogs);
         app.use(HEADLESS_LOG_DOWNLOAD_PATH_PREFIX, this.headlessLogController.headlessLogDownload);
+        app.use("/live", this.livenessController.apiRouter);
         app.use(this.newsletterSubscriptionController.apiRouter);
         app.use("/version", (req: express.Request, res: express.Response, next: express.NextFunction) => {
             res.send(this.config.version);