diff --git a/components/gitpod-db/src/project-db.ts b/components/gitpod-db/src/project-db.ts index 5f26d2e727694e..e5ad3302930601 100644 --- a/components/gitpod-db/src/project-db.ts +++ b/components/gitpod-db/src/project-db.ts @@ -4,7 +4,7 @@ * See License.enterprise.txt in the project root folder. */ -import { PartialProject, Project, ProjectEnvVar, ProjectEnvVarWithValue } from "@gitpod/gitpod-protocol"; +import { PartialProject, Project, ProjectEnvVar, ProjectEnvVarWithValue, ProjectUsage } from "@gitpod/gitpod-protocol"; export const ProjectDB = Symbol("ProjectDB"); export interface ProjectDB { @@ -30,4 +30,6 @@ export interface ProjectDB { getProjectEnvironmentVariableValues(envVars: ProjectEnvVar[]): Promise; findCachedProjectOverview(projectId: string): Promise; storeCachedProjectOverview(projectId: string, overview: Project.Overview): Promise; + getProjectUsage(projectId: string): Promise; + updateProjectUsage(projectId: string, usage: Partial): Promise; } diff --git a/components/gitpod-db/src/tables.ts b/components/gitpod-db/src/tables.ts index 2d7b8c45a3f035..47a46baf1588a2 100644 --- a/components/gitpod-db/src/tables.ts +++ b/components/gitpod-db/src/tables.ts @@ -256,6 +256,12 @@ export class GitpodTableDescriptionProvider implements TableDescriptionProvider deletionColumn: "deleted", timeColumn: "_lastModified", }, + { + name: "d_b_project_usage", + primaryKeys: ["projectId"], + deletionColumn: "deleted", + timeColumn: "_lastModified", + }, /** * BEWARE * diff --git a/components/gitpod-db/src/typeorm/deleted-entry-gc.ts b/components/gitpod-db/src/typeorm/deleted-entry-gc.ts index e7b6225dfbd7e0..e25f00f4047222 100644 --- a/components/gitpod-db/src/typeorm/deleted-entry-gc.ts +++ b/components/gitpod-db/src/typeorm/deleted-entry-gc.ts @@ -61,6 +61,7 @@ const tables: TableWithDeletion[] = [ { deletionColumn: "deleted", name: "d_b_oss_allow_list" }, { deletionColumn: "deleted", name: "d_b_project_env_var" }, { deletionColumn: "deleted", name: "d_b_project_info" }, + { deletionColumn: "deleted", name: "d_b_project_usage" }, ]; interface TableWithDeletion { diff --git a/components/gitpod-db/src/typeorm/entity/db-project-usage.ts b/components/gitpod-db/src/typeorm/entity/db-project-usage.ts new file mode 100644 index 00000000000000..4013bcecab0458 --- /dev/null +++ b/components/gitpod-db/src/typeorm/entity/db-project-usage.ts @@ -0,0 +1,26 @@ +/** + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. + * Licensed under the Gitpod Enterprise Source Code License, + * See License.enterprise.txt in the project root folder. + */ + +import { Entity, Column, PrimaryColumn } from "typeorm"; + +import { TypeORM } from "../../typeorm/typeorm"; + +@Entity() +// on DB but not Typeorm: @Index("ind_dbsync", ["_lastModified"]) // DBSync +export class DBProjectUsage { + @PrimaryColumn(TypeORM.UUID_COLUMN_TYPE) + projectId: string; + + @Column("varchar") + lastWebhookReceived: string; + + @Column("varchar") + lastWorkspaceStart: string; + + // This column triggers the db-sync deletion mechanism. It's not intended for public consumption. + @Column() + deleted: boolean; +} diff --git a/components/gitpod-db/src/typeorm/migration/1649667202321-ProjectUsage.ts b/components/gitpod-db/src/typeorm/migration/1649667202321-ProjectUsage.ts new file mode 100644 index 00000000000000..d6378b015a9013 --- /dev/null +++ b/components/gitpod-db/src/typeorm/migration/1649667202321-ProjectUsage.ts @@ -0,0 +1,17 @@ +/** + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class ProjectUsage1649667202321 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + "CREATE TABLE IF NOT EXISTS `d_b_project_usage` (`projectId` char(36) NOT NULL, `lastWebhookReceived` varchar(255) NOT NULL DEFAULT '', `lastWorkspaceStart` varchar(255) NOT NULL DEFAULT '', `deleted` tinyint(4) NOT NULL DEFAULT '0', `_lastModified` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (`projectId`), KEY `ind_dbsync` (`_lastModified`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;", + ); + } + + public async down(queryRunner: QueryRunner): Promise {} +} diff --git a/components/gitpod-db/src/typeorm/project-db-impl.ts b/components/gitpod-db/src/typeorm/project-db-impl.ts index 858683818682f3..cd066eef9475ce 100644 --- a/components/gitpod-db/src/typeorm/project-db-impl.ts +++ b/components/gitpod-db/src/typeorm/project-db-impl.ts @@ -8,12 +8,13 @@ import { inject, injectable } from "inversify"; import { TypeORM } from "./typeorm"; import { Repository } from "typeorm"; import { v4 as uuidv4 } from "uuid"; -import { PartialProject, Project, ProjectEnvVar, ProjectEnvVarWithValue } from "@gitpod/gitpod-protocol"; +import { PartialProject, Project, ProjectEnvVar, ProjectEnvVarWithValue, ProjectUsage } from "@gitpod/gitpod-protocol"; import { EncryptionService } from "@gitpod/gitpod-protocol/lib/encryption/encryption-service"; import { ProjectDB } from "../project-db"; import { DBProject } from "./entity/db-project"; import { DBProjectEnvVar } from "./entity/db-project-env-vars"; import { DBProjectInfo } from "./entity/db-project-info"; +import { DBProjectUsage } from "./entity/db-project-usage"; function toProjectEnvVar(envVarWithValue: ProjectEnvVarWithValue): ProjectEnvVar { const envVar = { ...envVarWithValue }; @@ -42,6 +43,10 @@ export class ProjectDBImpl implements ProjectDB { return (await this.getEntityManager()).getRepository(DBProjectInfo); } + protected async getProjectUsageRepo(): Promise> { + return (await this.getEntityManager()).getRepository(DBProjectUsage); + } + public async findProjectById(projectId: string): Promise { const repo = await this.getRepo(); return repo.findOne({ id: projectId, markedDeleted: false }); @@ -146,6 +151,11 @@ export class ProjectDBImpl implements ProjectDB { if (info) { await projectInfoRepo.update(projectId, { deleted: true }); } + const projectUsageRepo = await this.getProjectUsageRepo(); + const usage = await projectUsageRepo.findOne({ projectId, deleted: false }); + if (usage) { + await projectUsageRepo.update(projectId, { deleted: true }); + } } public async setProjectEnvironmentVariable( @@ -229,4 +239,23 @@ export class ProjectDBImpl implements ProjectDB { creationTime: new Date().toISOString(), }); } + + public async getProjectUsage(projectId: string): Promise { + const projectUsageRepo = await this.getProjectUsageRepo(); + const usage = await projectUsageRepo.findOne({ projectId }); + if (usage) { + return { + lastWebhookReceived: usage.lastWebhookReceived, + lastWorkspaceStart: usage.lastWorkspaceStart, + }; + } + } + + public async updateProjectUsage(projectId: string, usage: Partial): Promise { + const projectUsageRepo = await this.getProjectUsageRepo(); + await projectUsageRepo.save({ + projectId, + ...usage, + }); + } } diff --git a/components/gitpod-protocol/src/teams-projects-protocol.ts b/components/gitpod-protocol/src/teams-projects-protocol.ts index b4704ee799dffb..67d9a83750084e 100644 --- a/components/gitpod-protocol/src/teams-projects-protocol.ts +++ b/components/gitpod-protocol/src/teams-projects-protocol.ts @@ -69,6 +69,11 @@ export namespace Project { export type PartialProject = DeepPartial & Pick; +export interface ProjectUsage { + lastWebhookReceived: string; + lastWorkspaceStart: string; +} + export interface PrebuildWithStatus { info: PrebuildInfo; status: PrebuiltWorkspaceState; diff --git a/components/server/ee/src/prebuilds/bitbucket-app.ts b/components/server/ee/src/prebuilds/bitbucket-app.ts index ae7d5fcb5766ca..c73ae2f817ce2b 100644 --- a/components/server/ee/src/prebuilds/bitbucket-app.ts +++ b/components/server/ee/src/prebuilds/bitbucket-app.ts @@ -95,6 +95,14 @@ export class BitbucketApp { ): Promise { const span = TraceContext.startSpan("Bitbucket.handlePushHook", ctx); try { + const projectAndOwner = await this.findProjectAndOwner(data.gitCloneUrl, user); + if (projectAndOwner.project) { + /* tslint:disable-next-line */ + /** no await */ this.projectDB.updateProjectUsage(projectAndOwner.project.id, { + lastWebhookReceived: new Date().toISOString(), + }); + } + const contextURL = this.createContextUrl(data); const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; span.setTag("contextURL", contextURL); @@ -116,11 +124,10 @@ export class BitbucketApp { data.commitHash, ); } - const projectAndOwner = await this.findProjectAndOwner(data.gitCloneUrl, user); // todo@alex: add branch and project args const ws = await this.prebuildManager.startPrebuild( { span }, - { user, project: projectAndOwner?.project, context, commitInfo }, + { user, project: projectAndOwner.project, context, commitInfo }, ); return ws; } finally { diff --git a/components/server/ee/src/prebuilds/github-app.ts b/components/server/ee/src/prebuilds/github-app.ts index 38be4e8b2fded3..f2197bbe55c2a5 100644 --- a/components/server/ee/src/prebuilds/github-app.ts +++ b/components/server/ee/src/prebuilds/github-app.ts @@ -240,8 +240,14 @@ export class GithubApp { const installationId = ctx.payload.installation?.id; const cloneURL = ctx.payload.repository.clone_url; let { user, project } = await this.findOwnerAndProject(installationId, cloneURL); - const logCtx: LogContext = { userId: user.id }; + if (project) { + /* tslint:disable-next-line */ + /** no await */ this.projectDB.updateProjectUsage(project.id, { + lastWebhookReceived: new Date().toISOString(), + }); + } + const logCtx: LogContext = { userId: user.id }; if (!!user.blocked) { log.info(logCtx, `Blocked user tried to start prebuild`, { repo: ctx.payload.repository }); return; @@ -347,6 +353,12 @@ export class GithubApp { const pr = ctx.payload.pull_request; const contextURL = pr.html_url; let { user, project } = await this.findOwnerAndProject(installationId, cloneURL); + if (project) { + /* tslint:disable-next-line */ + /** no await */ this.projectDB.updateProjectUsage(project.id, { + lastWebhookReceived: new Date().toISOString(), + }); + } const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; const config = await this.prebuildManager.fetchConfig({ span }, user, context); diff --git a/components/server/ee/src/prebuilds/github-enterprise-app.ts b/components/server/ee/src/prebuilds/github-enterprise-app.ts index 625d0c38a08df8..a6cacb0eefe03c 100644 --- a/components/server/ee/src/prebuilds/github-enterprise-app.ts +++ b/components/server/ee/src/prebuilds/github-enterprise-app.ts @@ -116,6 +116,15 @@ export class GitHubEnterpriseApp { ): Promise { const span = TraceContext.startSpan("GitHubEnterpriseApp.handlePushHook", ctx); try { + const cloneURL = payload.repository.clone_url; + const projectAndOwner = await this.findProjectAndOwner(cloneURL, user); + if (projectAndOwner.project) { + /* tslint:disable-next-line */ + /** no await */ this.projectDB.updateProjectUsage(projectAndOwner.project.id, { + lastWebhookReceived: new Date().toISOString(), + }); + } + const contextURL = this.createContextUrl(payload); span.setTag("contextURL", contextURL); const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; @@ -127,15 +136,13 @@ export class GitHubEnterpriseApp { log.debug("GitHub Enterprise push event: Starting prebuild.", { contextURL }); - const cloneURL = payload.repository.clone_url; - const projectAndOwner = await this.findProjectAndOwner(cloneURL, user); const commitInfo = await this.getCommitInfo(user, payload.repository.url, payload.after); const ws = await this.prebuildManager.startPrebuild( { span }, { context, user: projectAndOwner.user, - project: projectAndOwner?.project, + project: projectAndOwner.project, commitInfo, }, ); diff --git a/components/server/ee/src/prebuilds/gitlab-app.ts b/components/server/ee/src/prebuilds/gitlab-app.ts index e49098435bea9f..ca87ef378f36a8 100644 --- a/components/server/ee/src/prebuilds/gitlab-app.ts +++ b/components/server/ee/src/prebuilds/gitlab-app.ts @@ -108,6 +108,13 @@ export class GitLabApp { span.setTag("contextURL", contextURL); const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext; const projectAndOwner = await this.findProjectAndOwner(context.repository.cloneUrl, user); + if (projectAndOwner.project) { + /* tslint:disable-next-line */ + /** no await */ this.projectDB.updateProjectUsage(projectAndOwner.project.id, { + lastWebhookReceived: new Date().toISOString(), + }); + } + const config = await this.prebuildManager.fetchConfig({ span }, user, context); if (!this.prebuildManager.shouldPrebuild(config)) { log.debug({ userId: user.id }, "GitLab push hook: There is no prebuild config.", { @@ -123,8 +130,8 @@ export class GitLabApp { const ws = await this.prebuildManager.startPrebuild( { span }, { - user: projectAndOwner?.user || user, - project: projectAndOwner?.project, + user: projectAndOwner.user || user, + project: projectAndOwner.project, context, commitInfo, }, diff --git a/components/server/ee/src/prebuilds/prebuild-manager.ts b/components/server/ee/src/prebuilds/prebuild-manager.ts index a80fea00e44e73..725af99596ceae 100644 --- a/components/server/ee/src/prebuilds/prebuild-manager.ts +++ b/components/server/ee/src/prebuilds/prebuild-manager.ts @@ -159,33 +159,11 @@ export class PrebuildManager { prebuildContext, context.normalizedContextURL!, ); - const prebuildPromise = this.workspaceDB.trace({ span }).findPrebuildByWorkspaceID(workspace.id)!; - - span.setTag("starting", true); - const projectEnvVars = await projectEnvVarsPromise; - await this.workspaceStarter.startWorkspace({ span }, workspace, user, [], projectEnvVars, { - excludeFeatureFlags: ["full_workspace_backup"], - }); - const prebuild = await prebuildPromise; + const prebuild = await this.workspaceDB.trace({ span }).findPrebuildByWorkspaceID(workspace.id)!; if (!prebuild) { throw new Error(`Failed to create a prebuild for: ${context.normalizedContextURL}`); } - if (await this.shouldRateLimitPrebuild(span, cloneURL)) { - prebuild.state = "aborted"; - prebuild.error = - "Prebuild is rate limited. Please contact Gitpod if you believe this happened in error."; - - await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild); - span.setTag("starting", false); - span.setTag("ratelimited", true); - return { - wsid: workspace.id, - prebuildId: prebuild.id, - done: false, - }; - } - if (project) { let aCommitInfo = commitInfo; if (!aCommitInfo) { @@ -205,6 +183,26 @@ export class PrebuildManager { } await this.storePrebuildInfo({ span }, project, prebuild, workspace, user, aCommitInfo); } + + if (await this.shouldRateLimitPrebuild(span, cloneURL)) { + prebuild.state = "aborted"; + prebuild.error = + "Prebuild is rate limited. Please contact Gitpod if you believe this happened in error."; + await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild); + span.setTag("ratelimited", true); + } else if (project && (await this.shouldSkipInactiveProject(project))) { + prebuild.state = "aborted"; + prebuild.error = + "Project is inactive. Please start a new workspace for this project to re-enable prebuilds."; + await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild); + } else { + span.setTag("starting", true); + const projectEnvVars = await projectEnvVarsPromise; + await this.workspaceStarter.startWorkspace({ span }, workspace, user, [], projectEnvVars, { + excludeFeatureFlags: ["full_workspace_backup"], + }); + } + return { prebuildId: prebuild.id, wsid: workspace.id, done: false }; } catch (err) { TraceContext.setError({ span }, err); @@ -347,4 +345,15 @@ export class PrebuildManager { // Last resort default return PREBUILD_LIMITER_DEFAULT_LIMIT; } + + private async shouldSkipInactiveProject(project: Project): Promise { + const usage = await this.projectService.getProjectUsage(project.id); + if (!usage?.lastWorkspaceStart) { + return false; + } + const now = Date.now(); + const lastUse = new Date(usage.lastWorkspaceStart).getTime(); + const inactiveProjectTime = 1000 * 60 * 60 * 24 * 7 * 10; // 10 weeks + return now - lastUse > inactiveProjectTime; + } } diff --git a/components/server/src/gitlab/api.ts b/components/server/src/gitlab/api.ts index e6fd6139325329..243e5b10170f2a 100644 --- a/components/server/src/gitlab/api.ts +++ b/components/server/src/gitlab/api.ts @@ -75,7 +75,7 @@ export class GitLabApi { log.error(`GitLab request error`, error); throw error; } finally { - log.info(`GitLab request took ${new Date().getTime() - before} ms`); + log.debug(`GitLab request took ${new Date().getTime() - before} ms`); } } diff --git a/components/server/src/projects/projects-service.ts b/components/server/src/projects/projects-service.ts index 941e34ffa6aa3b..07b28bb8791cd3 100644 --- a/components/server/src/projects/projects-service.ts +++ b/components/server/src/projects/projects-service.ts @@ -19,7 +19,7 @@ import { import { HostContextProvider } from "../auth/host-context-provider"; import { RepoURL } from "../repohost"; import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; -import { PartialProject } from "@gitpod/gitpod-protocol/src/teams-projects-protocol"; +import { PartialProject, ProjectUsage } from "@gitpod/gitpod-protocol/src/teams-projects-protocol"; @injectable() export class ProjectsService { @@ -248,4 +248,8 @@ export class ProjectsService { async deleteProjectEnvironmentVariable(variableId: string): Promise { return this.projectDB.deleteProjectEnvironmentVariable(variableId); } + + async getProjectUsage(projectId: string): Promise { + return this.projectDB.getProjectUsage(projectId); + } } diff --git a/components/server/src/user/user-controller.ts b/components/server/src/user/user-controller.ts index cfdb4b913d504b..01bd23a8c2d40c 100644 --- a/components/server/src/user/user-controller.ts +++ b/components/server/src/user/user-controller.ts @@ -580,7 +580,7 @@ export class UserController { await this.userService.updateUserEnvVarsOnLogin(user, envVars); await this.userService.acceptCurrentTerms(user); - /* no await */ trackSignup(user, req, this.analytics).catch((err) => + /** no await */ trackSignup(user, req, this.analytics).catch((err) => log.warn({ userId: user.id }, "trackSignup", err), ); diff --git a/components/server/src/workspace/workspace-starter.ts b/components/server/src/workspace/workspace-starter.ts index 1e0a9c585a520e..717de3be973bb7 100644 --- a/components/server/src/workspace/workspace-starter.ts +++ b/components/server/src/workspace/workspace-starter.ts @@ -206,6 +206,13 @@ export class WorkspaceStarter { const span = TraceContext.startSpan("WorkspaceStarter.startWorkspace", ctx); span.setTag("workspaceId", workspace.id); + if (workspace.projectId && workspace.type === "regular") { + /* tslint:disable-next-line */ + /** no await */ this.projectDB.updateProjectUsage(workspace.projectId, { + lastWorkspaceStart: new Date().toISOString(), + }); + } + options = options || {}; try { // Some workspaces do not have an image source.