diff --git a/WORKSPACE.yaml b/WORKSPACE.yaml index 25a43ac8c47f88..597a7ad3ab7bae 100644 --- a/WORKSPACE.yaml +++ b/WORKSPACE.yaml @@ -7,7 +7,7 @@ defaultArgs: jbMarketplacePublishTrigger: "false" publishToJBMarketplace: true localAppVersion: unknown - codeCommit: 06783e74552ddefd74c7e02cdcce19054b294469 + codeCommit: ca48b16256fad72bc239a27bde3dc78a369aafa6 codeQuality: stable intellijDownloadUrl: "https://download.jetbrains.com/idea/ideaIU-2022.2.1.tar.gz" golandDownloadUrl: "https://download.jetbrains.com/go/goland-2022.2.2.tar.gz" diff --git a/components/gitpod-db/src/typeorm/code-sync-resource-db.spec.db.ts b/components/gitpod-db/src/typeorm/code-sync-resource-db.spec.db.ts index 2826665fa890ad..c056d7726d6d35 100644 --- a/components/gitpod-db/src/typeorm/code-sync-resource-db.spec.db.ts +++ b/components/gitpod-db/src/typeorm/code-sync-resource-db.spec.db.ts @@ -23,7 +23,7 @@ export class CodeSyncResourceDBSpec { } async after(): Promise { - await this.db.delete(this.userId, () => Promise.resolve()); + await this.db.deleteSettingsSyncResources(this.userId, () => Promise.resolve()); } @test() @@ -120,7 +120,13 @@ export class CodeSyncResourceDBSpec { async roundRobinInsert(): Promise { const kind = "machines"; const expectation: string[] = []; - const doInsert = async (rev: string, oldRevs: string[]) => { + const doInsert = async (newRev: string, oldRevs?: string[]) => { + expectation.unshift(newRev); + + if (!oldRevs) { + return; + } + for (let rev of oldRevs) { await this.db.deleteResource(this.userId, kind, rev, async () => {}); } @@ -134,23 +140,23 @@ export class CodeSyncResourceDBSpec { await assertResources(); - expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!); - expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!); - expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!); + await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true }); + await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true }); + await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true }); await assertResources(); - expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!); + await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true }); expectation.length = revLimit; await assertResources(); - expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!); - expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!); + await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true }); + await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true }); expectation.length = revLimit; await assertResources(); - expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!); - expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!); - expectation.unshift((await this.db.insert(this.userId, kind, doInsert, { revLimit }))!); + await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true }); + await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true }); + await this.db.insert(this.userId, kind, doInsert, { revLimit, overwrite: true }); expectation.length = revLimit; await assertResources(); } diff --git a/components/gitpod-db/src/typeorm/code-sync-resource-db.ts b/components/gitpod-db/src/typeorm/code-sync-resource-db.ts index 7ded3f537d5d8d..ce321ecf390c0d 100644 --- a/components/gitpod-db/src/typeorm/code-sync-resource-db.ts +++ b/components/gitpod-db/src/typeorm/code-sync-resource-db.ts @@ -13,6 +13,7 @@ import { TypeORM } from "./typeorm"; export interface CodeSyncInsertOptions { latestRev?: string; revLimit?: number; + overwrite?: boolean; } @injectable() @@ -58,14 +59,14 @@ export class CodeSyncResourceDB { return this.doGetResources(connection.manager, userId, kind); } - async delete(userId: string, doDelete: () => Promise): Promise { + async deleteSettingsSyncResources(userId: string, doDelete: () => Promise): Promise { const connection = await this.typeORM.getConnection(); await connection.transaction(async (manager) => { await manager .createQueryBuilder() .update(DBCodeSyncResource) .set({ deleted: true }) - .where("userId = :userId AND deleted = 0", { userId }) + .where("userId = :userId AND kind != :kind AND deleted = 0", { userId, kind: "editSessions" }) .execute(); await doDelete(); }); @@ -74,37 +75,53 @@ export class CodeSyncResourceDB { async deleteResource( userId: string, kind: ServerResource, - rev: string, - doDelete: (rev: string) => Promise, + rev: string | undefined, + doDelete: (rev?: string) => Promise, ): Promise { const connection = await this.typeORM.getConnection(); - await connection.transaction(async (manager) => { - await manager - .createQueryBuilder() - .delete() - .from(DBCodeSyncResource) - .where("userId = :userId AND kind = :kind AND rev = :rev", { userId, kind, rev: rev }) - .execute(); - await doDelete(rev); - }); + if (rev) { + await connection.transaction(async (manager) => { + await manager + .createQueryBuilder() + .delete() + .from(DBCodeSyncResource) + .where("userId = :userId AND kind = :kind AND rev = :rev", { userId, kind, rev }) + .execute(); + await doDelete(rev); + }); + } else { + await connection.transaction(async (manager) => { + await manager + .createQueryBuilder() + .update(DBCodeSyncResource) + .set({ deleted: true }) + .where("userId = :userId AND kind = :kind", { userId, kind }) + .execute(); + await doDelete(); + }); + } } async insert( userId: string, kind: ServerResource, - doInsert: (rev: string, oldRevs: string[]) => Promise, + doInsert: (rev: string, oldRevs?: string[]) => Promise, params?: CodeSyncInsertOptions, ): Promise { const connection = await this.typeORM.getConnection(); return await connection.transaction(async (manager) => { let latest: DBCodeSyncResource | undefined; - let toDeleted: DBCodeSyncResource[] = []; + let toDeleted: DBCodeSyncResource[] | undefined; if (params?.revLimit) { const resources = await this.doGetResources(manager, userId, kind); latest = resources[0]; if (resources.length >= params.revLimit) { - // delete + 1 to insert instead of update - toDeleted = resources.splice(params?.revLimit - 1); + if (params.overwrite) { + // delete + 1 to insert instead of update + toDeleted = resources.splice(params?.revLimit - 1); + } else { + return undefined; + } } } else { latest = await this.doGetResource(manager, userId, kind, "latest"); @@ -122,7 +139,7 @@ export class CodeSyncResourceDB { .execute(); await doInsert( rev, - toDeleted.map((e) => e.rev), + toDeleted?.map((e) => e.rev), ); return rev; }); diff --git a/components/gitpod-db/src/typeorm/entity/db-code-sync-resource.ts b/components/gitpod-db/src/typeorm/entity/db-code-sync-resource.ts index d8269b5f818025..28975d2e0a169b 100644 --- a/components/gitpod-db/src/typeorm/entity/db-code-sync-resource.ts +++ b/components/gitpod-db/src/typeorm/entity/db-code-sync-resource.ts @@ -40,8 +40,8 @@ export interface IUserDataManifest { //readonly ref: string; } -export type ServerResource = SyncResource | "machines"; -export const ALL_SERVER_RESOURCES: ServerResource[] = [...ALL_SYNC_RESOURCES, "machines"]; +export type ServerResource = SyncResource | "machines" | "editSessions" | "profiles"; +export const ALL_SERVER_RESOURCES: ServerResource[] = [...ALL_SYNC_RESOURCES, "machines", "editSessions", "profiles"]; @Entity() @Index("ind_dbsync", ["created"]) // DBSync diff --git a/components/server/src/code-sync/code-sync-service.ts b/components/server/src/code-sync/code-sync-service.ts index 8c3ac118321e1e..f751f68863ea53 100644 --- a/components/server/src/code-sync/code-sync-service.ts +++ b/components/server/src/code-sync/code-sync-service.ts @@ -49,9 +49,16 @@ export type CodeSyncConfig = Partial<{ }; }>; -const objectPrefix = "code-sync/"; +function getNamePrefix(resource: ServerResource) { + if (resource === "editSessions") { + return "edit-sessions/"; + } else { + return "code-sync/"; + } +} + function toObjectName(resource: ServerResource, rev: string): string { - return objectPrefix + resource + "/" + rev; + return getNamePrefix(resource) + resource + "/" + rev; } const fromTheiaRev = "from-theia"; @@ -249,10 +256,11 @@ export class CodeSyncService { resourceKey === "machines" ? 1 : config.resources?.[resourceKey]?.revLimit || config?.revLimit || defaultRevLimit; + const isEditSessionsResource = resourceKey === "editSessions"; const userId = req.user.id; const contentType = req.headers["content-type"] || "*/*"; - let oldRevList: string[] = []; - const rev = await this.db.insert( + let oldRevList: string[] | undefined; + const newRev = await this.db.insert( userId, resourceKey, async (rev, oldRevs) => { @@ -282,15 +290,19 @@ export class CodeSyncService { } oldRevList = oldRevs; }, - { latestRev, revLimit }, + { latestRev, revLimit, overwrite: !isEditSessionsResource }, ); - // sync delete old revs from storage - this.deleteObjects(userId, resourceKey, oldRevList).catch((e) => {}); - if (!rev) { - res.sendStatus(412); + if (oldRevList && oldRevList.length > 0) { + // sync delete old revs from storage + Promise.allSettled(oldRevList.map((rev) => this.deleteResource(userId, resourceKey, rev))).catch( + () => {}, + ); + } + if (!newRev) { + res.sendStatus(isEditSessionsResource ? 400 : 412); return; } - res.setHeader("etag", rev); + res.setHeader("etag", newRev); res.sendStatus(200); return; }, @@ -300,11 +312,13 @@ export class CodeSyncService { res.sendStatus(204); return; } + + // This endpoint is used to delete settings-sync data only const userId = req.user.id; - await this.db.delete(userId, async () => { + await this.db.deleteSettingsSyncResources(userId, async () => { const request = new DeleteRequest(); request.setOwnerId(userId); - request.setPrefix(objectPrefix); + request.setPrefix(getNamePrefix("machines")); try { const blobsClient = this.blobsProvider.getDefault(); await util.promisify(blobsClient.delete.bind(blobsClient))(request); @@ -316,6 +330,24 @@ export class CodeSyncService { return; }); + router.delete("/v1/resource/:resource/:ref?", async (req, res) => { + if (!User.is(req.user)) { + res.sendStatus(204); + return; + } + + // This endpoint is used to delete edit sessions data only + const { resource, ref } = req.params; + if (resource !== "editSessions") { + res.sendStatus(400); + return; + } + + const userId = req.user.id; + await this.deleteResource(userId, resource, ref); + res.sendStatus(200); + }); + return router; } @@ -355,32 +387,35 @@ export class CodeSyncService { return JSON.stringify(extensions); } - protected async deleteObjects(userId: string, resourceKey: ServerResource, revs: string[]) { - const tasks = revs.map((rev) => - this.db - .deleteResource(userId, resourceKey, rev, async (rev: string) => { - const obj = toObjectName(resourceKey, rev); - try { - const request = new DeleteRequest(); - request.setOwnerId(userId); - request.setExact(obj); - const blobsClient = this.blobsProvider.getDefault(); - await util.promisify(blobsClient.delete.bind(blobsClient))( - request, - ); - } catch (err) { - if (err.code === status.NOT_FOUND) { - return; - } - log.error({ userId }, "code sync: failed to delete obj", err, { object: obj }); - throw err; + protected async deleteResource(userId: string, resourceKey: ServerResource, rev?: string) { + try { + await this.db.deleteResource(userId, resourceKey, rev, async (rev?: string) => { + try { + const request = new DeleteRequest(); + request.setOwnerId(userId); + if (rev) { + request.setExact(toObjectName(resourceKey, rev)); + } else { + request.setPrefix(getNamePrefix(resourceKey) + resourceKey); } - }) - .catch((err) => { - log.error({ userId }, "code sync: failed to delete", err); - }), - ); - await Promise.allSettled(tasks); - return; + const blobsClient = this.blobsProvider.getDefault(); + await util.promisify(blobsClient.delete.bind(blobsClient))(request); + } catch (e) { + if (e.code === status.NOT_FOUND) { + return; + } + throw e; + } + }); + } catch (e) { + if (rev) { + log.error({ userId }, "code sync: failed to delete obj", e, { + object: toObjectName(resourceKey, rev), + }); + } else { + log.error({ userId }, "code sync: failed to delete", e); + } + throw e; + } } }