Skip to content

[server] bump GitLab API library #13001

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 19, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions components/server/ee/src/gitlab/gitlab-app-support.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/**
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

import { User } from "@gitpod/gitpod-protocol";
import { skipIfEnvVarNotSet } from "@gitpod/gitpod-protocol/lib/util/skip-if";
import { expect } from "chai";
import { Container, ContainerModule } from "inversify";
import { suite, retries, test, timeout } from "mocha-typescript";
import { AuthProviderParams } from "../../../src/auth/auth-provider";
import { DevData } from "../../../src/dev/dev-data";
import { GitLabApi } from "../../../src/gitlab/api";
import { GitLabTokenHelper } from "../../../src/gitlab/gitlab-token-helper";
import { TokenProvider } from "../../../src/user/token-provider";
import { GitLabAppSupport } from "./gitlab-app-support";

@suite(timeout(10000), retries(2), skipIfEnvVarNotSet("GITPOD_TEST_TOKEN_GITLAB"))
class TestGitLabAppSupport {
static readonly AUTH_HOST_CONFIG: Partial<AuthProviderParams> = {
id: "Public-GitLab",
type: "GitLab",
verified: true,
description: "",
icon: "",
host: "gitlab.com",
};

protected appSupport: GitLabAppSupport;
protected user: User;
protected container: Container;

public before() {
this.container = new Container();
this.container.load(
new ContainerModule((bind, unbind, isBound, rebind) => {
bind(GitLabApi).toSelf().inSingletonScope();
bind(AuthProviderParams).toConstantValue(TestGitLabAppSupport.AUTH_HOST_CONFIG);
bind(GitLabTokenHelper).toSelf().inSingletonScope();
bind(TokenProvider).toConstantValue(<TokenProvider>{
getTokenForHost: async () => DevData.createGitlabTestToken(),
getFreshPortAuthenticationToken: async (user: User, workspaceId: string) =>
DevData.createPortAuthTestToken(workspaceId),
});
bind(GitLabAppSupport).toSelf().inSingletonScope();
}),
);
this.appSupport = this.container.get(GitLabAppSupport);
this.user = DevData.createTestUser();
}

// this manual test is assume you've maintainer access to +200 project
// see createManyRepos in /workspace/gitpod/components/server/src/gitlab/gitlab-file-provider.spec.ts
// for how to set up a test group with that many projects ;-)
@test.skip public async testFindMoreThan200Projects() {
const result = await this.appSupport.getProviderRepositoriesForUser({
user: this.user,
provider: {
host: "gitlab.com",
authProviderId: "Public-GitLab",
authProviderType: "GitLab",
verified: true,
},
});
expect(result.length).to.be.greaterThan(200);
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

better this not-automated test, than nothing.

}
}

module.exports = new TestGitLabAppSupport();
27 changes: 7 additions & 20 deletions components/server/ee/src/gitlab/gitlab-app-support.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,11 @@
import { AuthProviderInfo, ProviderRepository, User } from "@gitpod/gitpod-protocol";
import { inject, injectable } from "inversify";
import { TokenProvider } from "../../../src/user/token-provider";
import { UserDB } from "@gitpod/gitpod-db/lib";
import { Gitlab } from "@gitbeaker/node";
import { ProjectSchemaDefault, NamespaceInfoSchemaDefault } from "@gitbeaker/core/dist/types/services/Projects";

// Add missing fields to Gitbeaker's ProjectSchema type
type ProjectSchema = ProjectSchemaDefault & {
last_activity_at: string;
namespace: NamespaceInfoSchemaDefault & {
avatar_url: string | null;
parent_id: number | null;
};
owner?: {
id: number;
name: string;
avatar_url: string | null;
};
};
import { GitLab } from "../../../src/gitlab/api";

@injectable()
export class GitLabAppSupport {
@inject(UserDB) protected readonly userDB: UserDB;
@inject(TokenProvider) protected readonly tokenProvider: TokenProvider;

async getProviderRepositoriesForUser(params: {
Expand All @@ -51,9 +35,12 @@ export class GitLabAppSupport {
// we are listing only those projects with access level of maintainers.
// also cf. https://docs.gitlab.com/ee/api/members.html#valid-access-levels
//
const projectsWithAccess = await api.Projects.all({ min_access_level: "40", perPage: 100 });
const projectsWithAccess = await api.Projects.all({
min_access_level: "40",
perPage: 100,
});
for (const project of projectsWithAccess) {
const aProject = project as ProjectSchema;
const aProject = project as GitLab.Project;
const path = aProject.path as string;
const fullPath = aProject.path_with_namespace as string;
const cloneUrl = aProject.http_url_to_repo as string;
Expand All @@ -77,7 +64,7 @@ export class GitLabAppSupport {
return result;
}

protected async getAccountAvatarUrl(project: ProjectSchema, providerHost: string): Promise<string> {
protected async getAccountAvatarUrl(project: GitLab.Project, providerHost: string): Promise<string> {
let owner = project.owner;
if (!owner && project.namespace && !project.namespace.parent_id) {
// Fall back to "root namespace" / "top-level group"
Expand Down
2 changes: 1 addition & 1 deletion components/server/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"/dist"
],
"dependencies": {
"@gitbeaker/node": "^25.6.0",
"@gitbeaker/node": "^35.7.0",
"@gitpod/content-service": "0.1.5",
"@gitpod/gitpod-db": "0.1.5",
"@gitpod/gitpod-messagebus": "0.1.5",
Expand Down
28 changes: 17 additions & 11 deletions components/server/src/gitlab/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ import {
Issues,
RepositoryFiles,
} from "@gitbeaker/core";
import { ProjectSchemaDefault } from "@gitbeaker/core/dist/types/services/Projects";
import { UserSchemaDefault } from "@gitbeaker/core/dist/types/services/Users";
import { ProjectExtendedSchema } from "@gitbeaker/core/dist/types/resources/Projects";
import { NamespaceSchema } from "@gitbeaker/core/dist/types/resources/Namespaces";
import { UserExtendedSchema, UserSchema } from "@gitbeaker/core/dist/types/resources/Users";
import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
import { GitLabScope } from "./scopes";
import { AuthProviderParams } from "../auth/auth-provider";
Expand Down Expand Up @@ -70,9 +71,11 @@ export class GitLabApi {
// "description": "404 Commit Not Found"
// }

return new GitLab.ApiError(`GitLab Request Error: ${error?.description}`, error);
return new GitLab.ApiError(
`GitLab Request Error: ${error?.description?.name || JSON.stringify(error?.description)}`,
error,
);
}
log.error(`GitLab request error`, error);
throw error;
} finally {
log.debug(`GitLab request took ${new Date().getTime() - before} ms`);
Expand All @@ -87,7 +90,9 @@ export class GitLabApi {
path: string,
): Promise<string | undefined> {
const projectId = `${org}/${name}`;
const result = await this.run<string>(user, (api) => api.RepositoryFiles.showRaw(projectId, path, commitish));
const result = await this.run<string>(user, (api) =>
api.RepositoryFiles.showRaw(projectId, path, { ref: commitish }),
);
if (GitLab.ApiError.is(result)) {
return undefined; // e.g. 404 error, because the file isn't found
}
Expand Down Expand Up @@ -132,14 +137,17 @@ export namespace GitLab {
/**
* https://github.com/gitlabhq/gitlabhq/blob/master/doc/api/projects.md#get-single-project
*/
export interface Project extends ProjectSchemaDefault {
export interface Project extends ProjectExtendedSchema {
visibility: "public" | "private" | "internal";
archived: boolean;
path: string; // "diaspora-project-site"
path_with_namespace: string; // "diaspora/diaspora-project-site"
creator_id: number;
owner: User;
permissions: Permissions;
namespace: Pick<
NamespaceSchema,
"id" | "name" | "path" | "kind" | "full_path" | "avatar_url" | "web_url" | "avatar_url" | "parent_id"
>;
owner: Pick<UserSchema, "id" | "name" | "created_at" | "avatar_url">;
merge_requests_enabled: boolean;
issues_enabled: boolean;
open_issues_count: number;
Expand Down Expand Up @@ -219,11 +227,9 @@ export namespace GitLab {
merge_requests_count: number;
}
// https://docs.gitlab.com/ee/api/users.html#list-current-user-for-normal-users
export interface User extends UserSchemaDefault {
export interface User extends UserExtendedSchema {
email: string;
state: "active" | string;
confirmed_at: string | undefined;
private_profile: boolean;
}
export interface Permissions {
project_access?: {
Expand Down
99 changes: 99 additions & 0 deletions components/server/src/gitlab/gitlab-file-provider.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/**
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
* Licensed under the GNU Affero General Public License (AGPL).
* See License-AGPL.txt in the project root for license information.
*/

import { User } from "@gitpod/gitpod-protocol";
import { skipIfEnvVarNotSet } from "@gitpod/gitpod-protocol/lib/util/skip-if";
import { expect } from "chai";
import { Container, ContainerModule } from "inversify";
import { suite, retries, test, timeout } from "mocha-typescript";
import { AuthProviderParams } from "../auth/auth-provider";
import { DevData } from "../dev/dev-data";
import { TokenProvider } from "../user/token-provider";
import { GitLab, GitLabApi } from "./api";
import { GitlabFileProvider } from "./file-provider";
import { GitLabTokenHelper } from "./gitlab-token-helper";

@suite(timeout(10000), retries(2), skipIfEnvVarNotSet("GITPOD_TEST_TOKEN_GITLAB"))
class TestFileProvider {
static readonly AUTH_HOST_CONFIG: Partial<AuthProviderParams> = {
id: "Public-GitLab",
type: "GitLab",
verified: true,
description: "",
icon: "",
host: "gitlab.com",
};

protected fileProvider: GitlabFileProvider;
protected user: User;
protected container: Container;

public before() {
this.container = new Container();
this.container.load(
new ContainerModule((bind, unbind, isBound, rebind) => {
bind(GitLabApi).toSelf().inSingletonScope();
bind(AuthProviderParams).toConstantValue(TestFileProvider.AUTH_HOST_CONFIG);
bind(GitLabTokenHelper).toSelf().inSingletonScope();
bind(TokenProvider).toConstantValue(<TokenProvider>{
getTokenForHost: async () => DevData.createGitlabTestToken(),
getFreshPortAuthenticationToken: async (user: User, workspaceId: string) =>
DevData.createPortAuthTestToken(workspaceId),
});
bind(GitlabFileProvider).toSelf().inSingletonScope();
}),
);
this.fileProvider = this.container.get(GitlabFileProvider);
this.user = DevData.createTestUser();
}

@test public async testFileContent() {
const result = await this.fileProvider.getFileContent(
{
repository: {
owner: "AlexTugarev",
name: "gp-test",
host: "gitlab.com",
cloneUrl: "unused in test",
},
revision: "af65de8b249855785bbc7a8073ebcf21f55bc8fb",
},
this.user,
"README.md",
);
expect(result).to.equal(`# gp-test

123`);
}

// manual test helper to create many repos
@test.skip public async createManyRepos() {
const api = this.container.get(GitLabApi);
for (let i = 151; i < 200; i++) {
try {
const project = await api.run<GitLab.Project>(this.user, (g) =>
g.Projects.create({
name: `test_project_${i}`,
path: `test_project_${i}`,
namespace_id: 57982169,
initialize_with_readme: true,
description: "generated project to test pagination in gitpod.io/new",
}),
);
if (GitLab.ApiError.is(project)) {
console.error(`attempt ${i} error: ${JSON.stringify(project.message)}`);
} else {
console.log(project.name_with_namespace + " created ✅");
}
await new Promise((resolve) => setTimeout(resolve, 500));
} catch (error) {
console.error(error);
}
}
}
}

module.exports = new TestFileProvider();
Loading