code-sync: store editSessions

WORKSPACE.yaml

@@ -7,7 +7,7 @@ defaultArgs:
   jbMarketplacePublishTrigger: "false"
   publishToJBMarketplace: true
   localAppVersion: unknown
-  codeCommit: 06783e74552ddefd74c7e02cdcce19054b294469
+  codeCommit: ca48b16256fad72bc239a27bde3dc78a369aafa6
   codeQuality: stable
   intellijDownloadUrl: "https://download.jetbrains.com/idea/ideaIU-2022.2.1.tar.gz"
   golandDownloadUrl: "https://download.jetbrains.com/go/goland-2022.2.2.tar.gz"

components/gitpod-db/src/typeorm/code-sync-resource-db.spec.db.ts

@@ -23,7 +23,7 @@ export class CodeSyncResourceDBSpec {
     }
 
     async after(): Promise<void> {
-        await this.db.delete(this.userId, () => Promise.resolve());
+        await this.db.deleteSettingsSyncResources(this.userId, () => Promise.resolve());
     }
 
     @test()
@@ -120,7 +120,13 @@ export class CodeSyncResourceDBSpec {
     async roundRobinInsert(): Promise<void> {
         const kind = "machines";
         const expectation: string[] = [];
-        const doInsert = async (rev: string, oldRevs: string[]) => {
+        const doInsert = async (newRev: string, oldRevs?: string[]) => {
+            expectation.unshift(newRev);
+
+            if (!oldRevs) {
+                return;
+            }
+
             for (let rev of oldRevs) {
                 await this.db.deleteResource(this.userId, kind, rev, async () => {});
             }
@@ -134,23 +140,23 @@ export class CodeSyncResourceDBSpec {
 
         await assertResources();
 
-        expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!);
-        expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!);
-        expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!);
+        await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true });
+        await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true });
+        await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true });
         await assertResources();
 
-        expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!);
+        await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true });
         expectation.length = revLimit;
         await assertResources();
 
-        expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!);
-        expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!);
+        await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true });
+        await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true });
         expectation.length = revLimit;
         await assertResources();
 
-        expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!);
-        expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!);
-        expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!);
+        await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true });
+        await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true });
+        await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true });
         expectation.length = revLimit;
         await assertResources();
     }

components/gitpod-db/src/typeorm/code-sync-resource-db.ts

@@ -13,6 +13,7 @@ import { TypeORM } from "./typeorm";
 export interface CodeSyncInsertOptions {
     latestRev?: string;
     revLimit?: number;
+    overwrite?: boolean;
 }
 
 @injectable()
@@ -58,14 +59,14 @@ export class CodeSyncResourceDB {
         return this.doGetResources(connection.manager, userId, kind);
     }
 
-    async delete(userId: string, doDelete: () => Promise<void>): Promise<void> {
+    async deleteSettingsSyncResources(userId: string, doDelete: () => Promise<void>): Promise<void> {
         const connection = await this.typeORM.getConnection();
         await connection.transaction(async (manager) => {
             await manager
                 .createQueryBuilder()
                 .update(DBCodeSyncResource)
                 .set({ deleted: true })
-                .where("userId = :userId AND deleted = 0", { userId })
+                .where("userId = :userId AND kind != :kind AND deleted = 0", { userId, kind: "editSessions" })
                 .execute();
             await doDelete();
         });
@@ -74,37 +75,53 @@ export class CodeSyncResourceDB {
     async deleteResource(
         userId: string,
         kind: ServerResource,
-        rev: string,
-        doDelete: (rev: string) => Promise<void>,
+        rev: string | undefined,
+        doDelete: (rev?: string) => Promise<void>,
     ): Promise<void> {
         const connection = await this.typeORM.getConnection();
-        await connection.transaction(async (manager) => {
-            await manager
-                .createQueryBuilder()
-                .delete()
-                .from(DBCodeSyncResource)
-                .where("userId = :userId AND kind = :kind AND rev = :rev", { userId, kind, rev: rev })
-                .execute();
-            await doDelete(rev);
-        });
+        if (rev) {
+            await connection.transaction(async (manager) => {
+                await manager
+                    .createQueryBuilder()
+                    .delete()
+                    .from(DBCodeSyncResource)
+                    .where("userId = :userId AND kind = :kind AND rev = :rev", { userId, kind, rev })
+                    .execute();
+                await doDelete(rev);
+            });
+        } else {
+            await connection.transaction(async (manager) => {
+                await manager
+                    .createQueryBuilder()
+                    .update(DBCodeSyncResource)
+                    .set({ deleted: true })
+                    .where("userId = :userId AND kind = :kind", { userId, kind })
+                    .execute();
+                await doDelete();
+            });
+        }
     }
 
     async insert(
         userId: string,
         kind: ServerResource,
-        doInsert: (rev: string, oldRevs: string[]) => Promise<void>,
+        doInsert: (rev: string, oldRevs?: string[]) => Promise<void>,
         params?: CodeSyncInsertOptions,
     ): Promise<string | undefined> {
         const connection = await this.typeORM.getConnection();
         return await connection.transaction(async (manager) => {
             let latest: DBCodeSyncResource | undefined;
-            let toDeleted: DBCodeSyncResource[] = [];
+            let toDeleted: DBCodeSyncResource[] | undefined;
             if (params?.revLimit) {
                 const resources = await this.doGetResources(manager, userId, kind);
                 latest = resources[0];
                 if (resources.length >= params.revLimit) {
-                    // delete + 1 to insert instead of update
-                    toDeleted = resources.splice(params?.revLimit - 1);
+                    if (params.overwrite) {
+                        // delete + 1 to insert instead of update
+                        toDeleted = resources.splice(params?.revLimit - 1);
+                    } else {
+                        return undefined;
+                    }
                 }
             } else {
                 latest = await this.doGetResource(manager, userId, kind, "latest");
@@ -122,7 +139,7 @@ export class CodeSyncResourceDB {
                 .execute();
             await doInsert(
                 rev,
-                toDeleted.map((e) => e.rev),
+                toDeleted?.map((e) => e.rev),
             );
             return rev;
         });

components/gitpod-db/src/typeorm/entity/db-code-sync-resource.ts

@@ -40,8 +40,8 @@ export interface IUserDataManifest {
     //readonly ref: string;
 }
 
-export type ServerResource = SyncResource | "machines";
-export const ALL_SERVER_RESOURCES: ServerResource[] = [...ALL_SYNC_RESOURCES, "machines"];
+export type ServerResource = SyncResource | "machines" | "editSessions" | "profiles";
+export const ALL_SERVER_RESOURCES: ServerResource[] = [...ALL_SYNC_RESOURCES, "machines", "editSessions", "profiles"];
 
 @Entity()
 @Index("ind_dbsync", ["created"]) // DBSync

components/server/src/code-sync/code-sync-service.ts

@@ -49,9 +49,16 @@ export type CodeSyncConfig = Partial<{
     };
 }>;
 
-const objectPrefix = "code-sync/";
+function getNamePrefix(resource: ServerResource) {
+    if (resource === "editSessions") {
+        return "edit-sessions/";
+    } else {
+        return "code-sync/";
+    }
+}
+
 function toObjectName(resource: ServerResource, rev: string): string {
-    return objectPrefix + resource + "/" + rev;
+    return getNamePrefix(resource) + resource + "/" + rev;
 }
 
 const fromTheiaRev = "from-theia";
@@ -249,10 +256,11 @@ export class CodeSyncService {
                     resourceKey === "machines"
                         ? 1
                         : config.resources?.[resourceKey]?.revLimit || config?.revLimit || defaultRevLimit;
+                const isEditSessionsResource = resourceKey === "editSessions";
                 const userId = req.user.id;
                 const contentType = req.headers["content-type"] || "*/*";
-                let oldRevList: string[] = [];
-                const rev = await this.db.insert(
+                let oldRevList: string[] | undefined;
+                const newRev = await this.db.insert(
                     userId,
                     resourceKey,
                     async (rev, oldRevs) => {
@@ -282,15 +290,19 @@ export class CodeSyncService {
                         }
                         oldRevList = oldRevs;
                     },
-                    { latestRev, revLimit },
+                    { latestRev, revLimit, overwrite: !isEditSessionsResource },
                 );
-                // sync delete old revs from storage
-                this.deleteObjects(userId, resourceKey, oldRevList).catch((e) => {});
-                if (!rev) {
-                    res.sendStatus(412);
+                if (oldRevList && oldRevList.length > 0) {
+                    // sync delete old revs from storage
+                    Promise.allSettled(oldRevList.map((rev) => this.deleteResource(userId, resourceKey, rev))).catch(
+                        () => {},
+                    );
+                }
+                if (!newRev) {
+                    res.sendStatus(isEditSessionsResource ? 400 : 412);
                     return;
                 }
-                res.setHeader("etag", rev);
+                res.setHeader("etag", newRev);
                 res.sendStatus(200);
                 return;
             },
@@ -300,11 +312,13 @@ export class CodeSyncService {
                 res.sendStatus(204);
                 return;
             }
+
+            // This endpoint is used to delete settings-sync data only
             const userId = req.user.id;
-            await this.db.delete(userId, async () => {
+            await this.db.deleteSettingsSyncResources(userId, async () => {
                 const request = new DeleteRequest();
                 request.setOwnerId(userId);
-                request.setPrefix(objectPrefix);
+                request.setPrefix(getNamePrefix("machines"));
                 try {
                     const blobsClient = this.blobsProvider.getDefault();
                     await util.promisify(blobsClient.delete.bind(blobsClient))(request);
@@ -316,6 +330,24 @@ export class CodeSyncService {
 
             return;
         });
+        router.delete("/v1/resource/:resource/:ref?", async (req, res) => {
+            if (!User.is(req.user)) {
+                res.sendStatus(204);
+                return;
+            }
+
+            // This endpoint is used to delete edit sessions data only
+            const { resource, ref } = req.params;
+            if (resource !== "editSessions") {
+                res.sendStatus(400);
+                return;
+            }
+
+            const userId = req.user.id;
+            await this.deleteResource(userId, resource, ref);
+            res.sendStatus(200);
+        });
+
         return router;
     }
 
@@ -355,32 +387,35 @@ export class CodeSyncService {
         return JSON.stringify(extensions);
     }
 
-    protected async deleteObjects(userId: string, resourceKey: ServerResource, revs: string[]) {
-        const tasks = revs.map((rev) =>
-            this.db
-                .deleteResource(userId, resourceKey, rev, async (rev: string) => {
-                    const obj = toObjectName(resourceKey, rev);
-                    try {
-                        const request = new DeleteRequest();
-                        request.setOwnerId(userId);
-                        request.setExact(obj);
-                        const blobsClient = this.blobsProvider.getDefault();
-                        await util.promisify<DeleteRequest, DeleteResponse>(blobsClient.delete.bind(blobsClient))(
-                            request,
-                        );
-                    } catch (err) {
-                        if (err.code === status.NOT_FOUND) {
-                            return;
-                        }
-                        log.error({ userId }, "code sync: failed to delete obj", err, { object: obj });
-                        throw err;
+    protected async deleteResource(userId: string, resourceKey: ServerResource, rev?: string) {
+        try {
+            await this.db.deleteResource(userId, resourceKey, rev, async (rev?: string) => {
+                try {
+                    const request = new DeleteRequest();
+                    request.setOwnerId(userId);
+                    if (rev) {
+                        request.setExact(toObjectName(resourceKey, rev));
+                    } else {
+                        request.setPrefix(getNamePrefix(resourceKey) + resourceKey);
                     }
-                })
-                .catch((err) => {
-                    log.error({ userId }, "code sync: failed to delete", err);
-                }),
-        );
-        await Promise.allSettled(tasks);
-        return;
+                    const blobsClient = this.blobsProvider.getDefault();
+                    await util.promisify<DeleteRequest, DeleteResponse>(blobsClient.delete.bind(blobsClient))(request);
+                } catch (e) {
+                    if (e.code === status.NOT_FOUND) {
+                        return;
+                    }
+                    throw e;
+                }
+            });
+        } catch (e) {
+            if (rev) {
+                log.error({ userId }, "code sync: failed to delete obj", e, {
+                    object: toObjectName(resourceKey, rev),
+                });
+            } else {
+                log.error({ userId }, "code sync: failed to delete", e);
+            }
+            throw e;
+        }
     }
 }