Skip to content

Commit 815b525

Browse files
jankeromnesjankeromnes
committed
Stop running prebuilds for inactive projects (10+ weeks)
Fixes #8911
1 parent 976feac commit 815b525

File tree

13 files changed

+159
-15
lines changed

13 files changed

+159
-15
lines changed

‎components/gitpod-db/src/project-db.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* See License.enterprise.txt in the project root folder.
55
*/
66

7-
import { PartialProject, Project, ProjectEnvVar, ProjectEnvVarWithValue } from "@gitpod/gitpod-protocol";
7+
import { PartialProject, Project, ProjectEnvVar, ProjectEnvVarWithValue, ProjectUsage } from "@gitpod/gitpod-protocol";
88

99
export const ProjectDB = Symbol("ProjectDB");
1010
export interface ProjectDB {
@@ -30,4 +30,6 @@ export interface ProjectDB {
3030
getProjectEnvironmentVariableValues(envVars: ProjectEnvVar[]): Promise<ProjectEnvVarWithValue[]>;
3131
findCachedProjectOverview(projectId: string): Promise<Project.Overview | undefined>;
3232
storeCachedProjectOverview(projectId: string, overview: Project.Overview): Promise<void>;
33+
getProjectUsage(projectId: string): Promise<ProjectUsage | undefined>;
34+
updateProjectUsage(projectId: string, usage: Partial<ProjectUsage>): Promise<void>;
3335
}

‎components/gitpod-db/src/typeorm/entity/db-project-usage.ts

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/**
2+
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
3+
* Licensed under the Gitpod Enterprise Source Code License,
4+
* See License.enterprise.txt in the project root folder.
5+
*/
6+
7+
import { Entity, Column, PrimaryColumn } from "typeorm";
8+
9+
import { TypeORM } from "../../typeorm/typeorm";
10+
11+
@Entity()
12+
// on DB but not Typeorm: @Index("ind_dbsync", ["_lastModified"]) // DBSync
13+
export class DBProjectUsage {
14+
@PrimaryColumn(TypeORM.UUID_COLUMN_TYPE)
15+
projectId: string;
16+
17+
@Column("varchar")
18+
lastWebhookReceived: string;
19+
20+
@Column("varchar")
21+
lastWorkspaceStart: string;
22+
23+
// This column triggers the db-sync deletion mechanism. It's not intended for public consumption.
24+
@Column()
25+
deleted: boolean;
26+
}

‎components/gitpod-db/src/typeorm/migration/1649667202321-ProjectUsage.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/**
2+
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
3+
* Licensed under the GNU Affero General Public License (AGPL).
4+
* See License-AGPL.txt in the project root for license information.
5+
*/
6+
7+
import { MigrationInterface, QueryRunner } from "typeorm";
8+
9+
export class ProjectUsage1649667202321 implements MigrationInterface {
10+
public async up(queryRunner: QueryRunner): Promise<void> {
11+
await queryRunner.query(
12+
"CREATE TABLE IF NOT EXISTS `d_b_project_usage` ( `projectId` char(36) NOT NULL, `lastWebhookReceived` varchar(255) NOT NULL, `lastWorkspaceStart` varchar(255) NOT NULL, `deleted` tinyint(4) NOT NULL DEFAULT '0', `_lastModified` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (`projectId`), KEY `ind_dbsync` (`_lastModified`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;",
13+
);
14+
}
15+
16+
public async down(queryRunner: QueryRunner): Promise<void> {}
17+
}

‎components/gitpod-db/src/typeorm/project-db-impl.ts

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@ import { inject, injectable } from "inversify";
88
import { TypeORM } from "./typeorm";
99
import { Repository } from "typeorm";
1010
import { v4 as uuidv4 } from "uuid";
11-
import { PartialProject, Project, ProjectEnvVar, ProjectEnvVarWithValue } from "@gitpod/gitpod-protocol";
11+
import { PartialProject, Project, ProjectEnvVar, ProjectEnvVarWithValue, ProjectUsage } from "@gitpod/gitpod-protocol";
1212
import { EncryptionService } from "@gitpod/gitpod-protocol/lib/encryption/encryption-service";
1313
import { ProjectDB } from "../project-db";
1414
import { DBProject } from "./entity/db-project";
1515
import { DBProjectEnvVar } from "./entity/db-project-env-vars";
1616
import { DBProjectInfo } from "./entity/db-project-info";
17+
import { DBProjectUsage } from "./entity/db-project-usage";
1718

1819
function toProjectEnvVar(envVarWithValue: ProjectEnvVarWithValue): ProjectEnvVar {
1920
const envVar = { ...envVarWithValue };
@@ -42,6 +43,10 @@ export class ProjectDBImpl implements ProjectDB {
4243
return (await this.getEntityManager()).getRepository<DBProjectInfo>(DBProjectInfo);
4344
}
4445

46+
protected async getProjectUsageRepo(): Promise<Repository<DBProjectUsage>> {
47+
return (await this.getEntityManager()).getRepository<DBProjectUsage>(DBProjectUsage);
48+
}
49+
4550
public async findProjectById(projectId: string): Promise<Project | undefined> {
4651
const repo = await this.getRepo();
4752
return repo.findOne({ id: projectId, markedDeleted: false });
@@ -146,6 +151,11 @@ export class ProjectDBImpl implements ProjectDB {
146151
if (info) {
147152
await projectInfoRepo.update(projectId, { deleted: true });
148153
}
154+
const projectUsageRepo = await this.getProjectUsageRepo();
155+
const usage = await projectUsageRepo.findOne({ projectId, deleted: false });
156+
if (usage) {
157+
await projectUsageRepo.update(projectId, { deleted: true });
158+
}
149159
}
150160

151161
public async setProjectEnvironmentVariable(
@@ -229,4 +239,23 @@ export class ProjectDBImpl implements ProjectDB {
229239
creationTime: new Date().toISOString(),
230240
});
231241
}
242+
243+
public async getProjectUsage(projectId: string): Promise<ProjectUsage | undefined> {
244+
const projectUsageRepo = await this.getProjectUsageRepo();
245+
const usage = await projectUsageRepo.findOne({ projectId });
246+
if (usage) {
247+
return {
248+
lastWebhookReceived: usage.lastWebhookReceived,
249+
lastWorkspaceStart: usage.lastWorkspaceStart,
250+
};
251+
}
252+
}
253+
254+
public async updateProjectUsage(projectId: string, usage: Partial<ProjectUsage>): Promise<void> {
255+
const projectUsageRepo = await this.getProjectUsageRepo();
256+
await projectUsageRepo.save({
257+
projectId,
258+
...usage,
259+
});
260+
}
232261
}

‎components/gitpod-protocol/src/teams-projects-protocol.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,11 @@ export namespace Project {
6969

7070
export type PartialProject = DeepPartial<Project> & Pick<Project, "id">;
7171

72+
export interface ProjectUsage {
73+
lastWebhookReceived: string;
74+
lastWorkspaceStart: string;
75+
}
76+
7277
export interface PrebuildWithStatus {
7378
info: PrebuildInfo;
7479
status: PrebuiltWorkspaceState;

‎components/server/ee/src/prebuilds/bitbucket-app.ts

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,14 @@ export class BitbucketApp {
9595
): Promise<StartPrebuildResult | undefined> {
9696
const span = TraceContext.startSpan("Bitbucket.handlePushHook", ctx);
9797
try {
98+
const projectAndOwner = await this.findProjectAndOwner(data.gitCloneUrl, user);
99+
if (projectAndOwner.project) {
100+
/* tslint:disable-next-line */
101+
/** no await */ this.projectDB.updateProjectUsage(projectAndOwner.project.id, {
102+
lastWebhookReceived: new Date().toISOString(),
103+
});
104+
}
105+
98106
const contextURL = this.createContextUrl(data);
99107
const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext;
100108
span.setTag("contextURL", contextURL);
@@ -116,11 +124,10 @@ export class BitbucketApp {
116124
data.commitHash,
117125
);
118126
}
119-
const projectAndOwner = await this.findProjectAndOwner(data.gitCloneUrl, user);
120127
// todo@alex: add branch and project args
121128
const ws = await this.prebuildManager.startPrebuild(
122129
{ span },
123-
{ user, project: projectAndOwner?.project, context, commitInfo },
130+
{ user, project: projectAndOwner.project, context, commitInfo },
124131
);
125132
return ws;
126133
} finally {

‎components/server/ee/src/prebuilds/github-app.ts

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,8 +240,14 @@ export class GithubApp {
240240
const installationId = ctx.payload.installation?.id;
241241
const cloneURL = ctx.payload.repository.clone_url;
242242
let { user, project } = await this.findOwnerAndProject(installationId, cloneURL);
243-
const logCtx: LogContext = { userId: user.id };
243+
if (project) {
244+
/* tslint:disable-next-line */
245+
/** no await */ this.projectDB.updateProjectUsage(project.id, {
246+
lastWebhookReceived: new Date().toISOString(),
247+
});
248+
}
244249

250+
const logCtx: LogContext = { userId: user.id };
245251
if (!!user.blocked) {
246252
log.info(logCtx, `Blocked user tried to start prebuild`, { repo: ctx.payload.repository });
247253
return;
@@ -347,6 +353,12 @@ export class GithubApp {
347353
const pr = ctx.payload.pull_request;
348354
const contextURL = pr.html_url;
349355
let { user, project } = await this.findOwnerAndProject(installationId, cloneURL);
356+
if (project) {
357+
/* tslint:disable-next-line */
358+
/** no await */ this.projectDB.updateProjectUsage(project.id, {
359+
lastWebhookReceived: new Date().toISOString(),
360+
});
361+
}
350362

351363
const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext;
352364
const config = await this.prebuildManager.fetchConfig({ span }, user, context);

‎components/server/ee/src/prebuilds/github-enterprise-app.ts

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,15 @@ export class GitHubEnterpriseApp {
116116
): Promise<StartPrebuildResult | undefined> {
117117
const span = TraceContext.startSpan("GitHubEnterpriseApp.handlePushHook", ctx);
118118
try {
119+
const cloneURL = payload.repository.clone_url;
120+
const projectAndOwner = await this.findProjectAndOwner(cloneURL, user);
121+
if (projectAndOwner.project) {
122+
/* tslint:disable-next-line */
123+
/** no await */ this.projectDB.updateProjectUsage(projectAndOwner.project.id, {
124+
lastWebhookReceived: new Date().toISOString(),
125+
});
126+
}
127+
119128
const contextURL = this.createContextUrl(payload);
120129
span.setTag("contextURL", contextURL);
121130
const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext;
@@ -127,15 +136,13 @@ export class GitHubEnterpriseApp {
127136

128137
log.debug("GitHub Enterprise push event: Starting prebuild.", { contextURL });
129138

130-
const cloneURL = payload.repository.clone_url;
131-
const projectAndOwner = await this.findProjectAndOwner(cloneURL, user);
132139
const commitInfo = await this.getCommitInfo(user, payload.repository.url, payload.after);
133140
const ws = await this.prebuildManager.startPrebuild(
134141
{ span },
135142
{
136143
context,
137144
user: projectAndOwner.user,
138-
project: projectAndOwner?.project,
145+
project: projectAndOwner.project,
139146
commitInfo,
140147
},
141148
);

‎components/server/ee/src/prebuilds/gitlab-app.ts

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,13 @@ export class GitLabApp {
108108
span.setTag("contextURL", contextURL);
109109
const context = (await this.contextParser.handle({ span }, user, contextURL)) as CommitContext;
110110
const projectAndOwner = await this.findProjectAndOwner(context.repository.cloneUrl, user);
111+
if (projectAndOwner.project) {
112+
/* tslint:disable-next-line */
113+
/** no await */ this.projectDB.updateProjectUsage(projectAndOwner.project.id, {
114+
lastWebhookReceived: new Date().toISOString(),
115+
});
116+
}
117+
111118
const config = await this.prebuildManager.fetchConfig({ span }, user, context);
112119
if (!this.prebuildManager.shouldPrebuild(config)) {
113120
log.debug({ userId: user.id }, "GitLab push hook: There is no prebuild config.", {
@@ -123,8 +130,8 @@ export class GitLabApp {
123130
const ws = await this.prebuildManager.startPrebuild(
124131
{ span },
125132
{
126-
user: projectAndOwner?.user || user,
127-
project: projectAndOwner?.project,
133+
user: projectAndOwner.user || user,
134+
project: projectAndOwner.project,
128135
context,
129136
commitInfo,
130137
},

‎components/server/ee/src/prebuilds/prebuild-manager.ts

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -171,10 +171,9 @@ export class PrebuildManager {
171171
throw new Error(`Failed to create a prebuild for: ${context.normalizedContextURL}`);
172172
}
173173

174-
if (await this.shouldRateLimitPrebuild(span, cloneURL)) {
174+
const cancelPrebuild = async (message: string) => {
175175
prebuild.state = "aborted";
176-
prebuild.error =
177-
"Prebuild is rate limited. Please contact Gitpod if you believe this happened in error.";
176+
prebuild.error = message;
178177

179178
await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild);
180179
span.setTag("starting", false);
@@ -184,9 +183,20 @@ export class PrebuildManager {
184183
prebuildId: prebuild.id,
185184
done: false,
186185
};
186+
};
187+
188+
if (await this.shouldRateLimitPrebuild(span, cloneURL)) {
189+
return await cancelPrebuild(
190+
"Prebuild is rate limited. Please contact Gitpod if you believe this happened in error.",
191+
);
187192
}
188193

189194
if (project) {
195+
if (await this.shouldSkipInactiveProject(project)) {
196+
return await cancelPrebuild(
197+
"Project is inactive. Please start a new workspace for this project to re-enable prebuilds.",
198+
);
199+
}
190200
let aCommitInfo = commitInfo;
191201
if (!aCommitInfo) {
192202
aCommitInfo = await getCommitInfo(
@@ -347,4 +357,15 @@ export class PrebuildManager {
347357
// Last resort default
348358
return PREBUILD_LIMITER_DEFAULT_LIMIT;
349359
}
360+
361+
private async shouldSkipInactiveProject(project: Project): Promise<boolean> {
362+
const usage = await this.projectService.getProjectUsage(project.id);
363+
if (!usage?.lastWorkspaceStart) {
364+
return false;
365+
}
366+
const now = Date.now();
367+
const lastUse = new Date(usage.lastWorkspaceStart).getTime();
368+
const inactiveProjectTime = 1000 * 60 * 60 * 24 * 7 * 10; // 10 weeks
369+
return now - lastUse > inactiveProjectTime;
370+
}
350371
}

0 commit comments

Comments
 (0)