[dashboard][server] Make Project Overview page faster by pre-fetching and caching Git provider data (branches)

Author: jankeromnes

components/gitpod-db/src/project-db.ts

@@ -21,4 +21,6 @@ export interface ProjectDB {
     getProjectEnvironmentVariableById(variableId: string): Promise<ProjectEnvVar | undefined>;
     deleteProjectEnvironmentVariable(variableId: string): Promise<void>;
     getProjectEnvironmentVariableValues(envVars: ProjectEnvVar[]): Promise<ProjectEnvVarWithValue[]>;
+    findCachedProjectOverview(projectId: string): Promise<Project.Overview | undefined>;
+    storeCachedProjectOverview(projectId: string, overview: Project.Overview): Promise<void>;
 }

components/gitpod-db/src/tables.ts

@@ -251,6 +251,12 @@ export class GitpodTableDescriptionProvider implements TableDescriptionProvider
             deletionColumn: 'deleted',
             timeColumn: '_lastModified',
         },
+        {
+            name: 'd_b_project_info',
+            primaryKeys: ['projectId'],
+            deletionColumn: 'deleted',
+            timeColumn: '_lastModified',
+        },
         /**
          * BEWARE
          *

components/gitpod-db/src/typeorm/deleted-entry-gc.ts

@@ -59,6 +59,7 @@ const tables: TableWithDeletion[] = [
     { deletionColumn: "deleted", name: "d_b_prebuild_info" },
     { deletionColumn: "deleted", name: "d_b_oss_allow_list" },
     { deletionColumn: "deleted", name: "d_b_project_env_var" },
+    { deletionColumn: "deleted", name: "d_b_project_info" },
 ];
 
 interface TableWithDeletion {

components/gitpod-db/src/typeorm/entity/db-prebuild-info-entry.ts

@@ -10,7 +10,7 @@ import { PrebuildInfo } from "@gitpod/gitpod-protocol";
 import { TypeORM } from "../../typeorm/typeorm";
 
 @Entity()
-export class DBPrebuildInfo  {
+export class DBPrebuildInfo {
 
     @PrimaryColumn(TypeORM.UUID_COLUMN_TYPE)
     prebuildId: string;

components/gitpod-db/src/typeorm/entity/db-project-info.ts

@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+ * Licensed under the Gitpod Enterprise Source Code License,
+ * See License.enterprise.txt in the project root folder.
+ */
+
+import { Entity, Column, PrimaryColumn } from "typeorm";
+import { Project } from "@gitpod/gitpod-protocol";
+
+import { TypeORM } from "../../typeorm/typeorm";
+
+@Entity()
+// on DB but not Typeorm: @Index("ind_dbsync", ["_lastModified"])   // DBSync
+export class DBProjectInfo {
+
+    @PrimaryColumn(TypeORM.UUID_COLUMN_TYPE)
+    projectId: string;
+
+    @Column({
+        type: 'simple-json',
+        transformer: (() => {
+            return {
+                to(value: any): any {
+                    return JSON.stringify(value);
+                },
+                from(value: any): any {
+                    try {
+                        const obj = JSON.parse(value);
+                        if (Project.Overview.is(obj)) {
+                            return obj;
+                        }
+                    } catch (error) {
+                    }
+                }
+            };
+        })()
+    })
+    overview: Project.Overview;
+
+    // This column triggers the db-sync deletion mechanism. It's not intended for public consumption.
+    @Column()
+    deleted: boolean;
+}

components/gitpod-db/src/typeorm/migration/1642497869312-ProjectInfo.ts

@@ -0,0 +1,18 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class ProjectInfo1642497869312 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query("CREATE TABLE IF NOT EXISTS `d_b_project_info` ( `projectId` char(36) NOT NULL, `overview` longtext NOT NULL, `deleted` tinyint(4) NOT NULL DEFAULT '0', `_lastModified` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (`projectId`), KEY `ind_dbsync` (`_lastModified`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;");
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+    }
+
+}

components/gitpod-db/src/typeorm/project-db-impl.ts

@@ -13,6 +13,7 @@ import { EncryptionService } from "@gitpod/gitpod-protocol/lib/encryption/encryp
 import { ProjectDB } from "../project-db";
 import { DBProject } from "./entity/db-project";
 import { DBProjectEnvVar } from "./entity/db-project-env-vars";
+import { DBProjectInfo } from "./entity/db-project-info";
 
 function toProjectEnvVar(envVarWithValue: ProjectEnvVarWithValue): ProjectEnvVar {
     const envVar = { ...envVarWithValue };
@@ -37,6 +38,10 @@ export class ProjectDBImpl implements ProjectDB {
         return (await this.getEntityManager()).getRepository<DBProjectEnvVar>(DBProjectEnvVar);
     }
 
+    protected async getProjectInfoRepo(): Promise<Repository<DBProjectInfo>> {
+        return (await this.getEntityManager()).getRepository<DBProjectInfo>(DBProjectInfo);
+    }
+
     public async findProjectById(projectId: string): Promise<Project | undefined> {
         const repo = await this.getRepo();
         return repo.findOne({ id: projectId, markedDeleted: false });
@@ -106,6 +111,12 @@ export class ProjectDBImpl implements ProjectDB {
             project.markedDeleted = true;
             await repo.save(project);
         }
+        // Delete any additional cached infos about this project
+        const projectInfoRepo = await this.getProjectInfoRepo();
+        const info = await projectInfoRepo.findOne({ projectId, deleted: false });
+        if (info) {
+            await projectInfoRepo.update(projectId, { deleted: true });
+        }
     }
 
     public async setProjectEnvironmentVariable(projectId: string, name: string, value: string, censored: boolean): Promise<void> {
@@ -164,4 +175,19 @@ export class ProjectDBImpl implements ProjectDB {
         const envVarsWithValues = await envVarRepo.findByIds(envVars);
         return envVarsWithValues;
     }
+
+    public async findCachedProjectOverview(projectId: string): Promise<Project.Overview | undefined> {
+        const projectInfoRepo = await this.getProjectInfoRepo();
+        const info = await projectInfoRepo.findOne({ projectId });
+        return info?.overview;
+    }
+
+    public async storeCachedProjectOverview(projectId: string, overview: Project.Overview): Promise<void> {
+        const projectInfoRepo = await this.getProjectInfoRepo();
+        await projectInfoRepo.save({
+            projectId,
+            overview,
+            creationTime: new Date().toISOString(),
+        });
+    }
 }

components/gitpod-protocol/src/teams-projects-protocol.ts

@@ -42,7 +42,13 @@ export namespace Project {
     }
 
     export interface Overview {
-        branches: BranchDetails[]
+        branches: BranchDetails[];
+    }
+
+    export namespace Overview {
+        export function is(data?: any): data is Project.Overview {
+            return Array.isArray(data?.branches);
+        }
     }
 
     export interface BranchDetails {

components/server/src/projects/projects-service.ts

@@ -37,9 +37,25 @@ export class ProjectsService {
         return this.projectDB.findProjectsByCloneUrls(cloneUrls);
     }
 
-    async getProjectOverview(user: User, project: Project): Promise<Project.Overview | undefined> {
-        const branches = await this.getBranchDetails(user, project);
-        return { branches };
+    async getProjectOverviewCached(user: User, project: Project): Promise<Project.Overview | undefined> {
+        // Check for a cached project overview (fast!)
+        const cachedPromise = this.projectDB.findCachedProjectOverview(project.id);
+
+        // ...but also refresh the cache on every request (asynchronously / in the background)
+        const refreshPromise = this.getBranchDetails(user, project).then(branches => {
+            const overview = { branches };
+            // No need to await here
+            this.projectDB.storeCachedProjectOverview(project.id, overview).catch(error => {
+                log.error(`Could not store cached project overview: ${error}`, { cloneUrl: project.cloneUrl })
+            });
+            return overview;
+        });
+
+        const cachedOverview = await cachedPromise;
+        if (cachedOverview) {
+            return cachedOverview;
+        }
+        return await refreshPromise;
     }
 
     protected getRepositoryProvider(project: Project) {
@@ -113,6 +129,10 @@ export class ProjectsService {
     }
 
     protected async onDidCreateProject(project: Project, installer: User) {
+        // Pre-fetch project details in the background -- don't await
+        this.getProjectOverviewCached(installer, project);
+
+        // Install the prebuilds webhook if possible
         let { userId, teamId, cloneUrl } = project;
         const parsedUrl = RepoURL.parseRepoUrl(project.cloneUrl);
         const hostContext = parsedUrl?.host ? this.hostContextProvider.get(parsedUrl?.host) : undefined;

components/server/src/workspace/gitpod-server-impl.ts

@@ -1677,7 +1677,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
         }
         await this.guardProjectOperation(user, projectId, "get");
         try {
-            return await this.projectsService.getProjectOverview(user, project);
+            return await this.projectsService.getProjectOverviewCached(user, project);
         } catch (error) {
             if (UnauthorizedError.is(error)) {
                 throw new ResponseError(ErrorCodes.NOT_AUTHENTICATED, "Unauthorized", error.data);