[server] Use workspace cluster as image-builder

Author: csweichel

components/image-builder-api/typescript/src/sugar.ts

@@ -15,13 +15,13 @@ import { BuildRequest, BuildResponse, BuildStatus, LogsRequest, LogsResponse, Re
 import { injectable, inject, optional } from 'inversify';
 import * as grpc from "@grpc/grpc-js";
 import { TextDecoder } from "util";
-import { ImageBuildLogInfo } from "@gitpod/gitpod-protocol";
+import { ImageBuildLogInfo, User, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol";
 
 export const ImageBuilderClientProvider = Symbol("ImageBuilderClientProvider");
 
 // ImageBuilderClientProvider caches image builder connections
 export interface ImageBuilderClientProvider {
-    getDefault(): PromisifiedImageBuilderClient
+    getDefault(user: User, workspace: Workspace, instance: WorkspaceInstance): Promise<PromisifiedImageBuilderClient>
 }
 
 function withTracing(ctx: TraceContext) {
@@ -53,7 +53,7 @@ export class CachingImageBuilderClientProvider implements ImageBuilderClientProv
     // Thus it makes sense to cache them rather than create a new connection for each request.
     protected connectionCache: PromisifiedImageBuilderClient | undefined;
 
-    getDefault() {
+    async getDefault(user: User, workspace: Workspace, instance: WorkspaceInstance) {
         let interceptors: grpc.Interceptor[] = [];
         if (this.clientCallMetrics) {
             interceptors = [ createClientCallMetricsInterceptor(this.clientCallMetrics) ];
@@ -78,6 +78,18 @@ export class CachingImageBuilderClientProvider implements ImageBuilderClientProv
         return connection;
     }
 
+    promisify(c: ImageBuilderClient): PromisifiedImageBuilderClient {
+        let interceptors: grpc.Interceptor[] = [];
+        if (this.clientCallMetrics) {
+            interceptors = [ createClientCallMetricsInterceptor(this.clientCallMetrics) ];
+        }
+
+        return new PromisifiedImageBuilderClient(
+            new ImageBuilderClient(this.clientConfig.address, grpc.credentials.createInsecure()),
+            interceptors
+        );
+    }
+
 }
 
 // StagedBuildResponse captures the multi-stage nature (starting, running, done) of image builds.

components/server/src/container-module.ts

@@ -99,6 +99,7 @@ import { ReferrerPrefixParser } from "./workspace/referrer-prefix-context-parser
 import { InstallationAdminTelemetryDataProvider } from "./installation-admin/telemetry-data-provider";
 import { IDEService } from "./ide-service";
 import { LicenseEvaluator } from "@gitpod/licensor/lib";
+import { WorkspaceClusterImagebuilderClientProvider } from "./workspace/workspace-cluster-imagebuilder-client-provider";
 
 export const productionContainerModule = new ContainerModule((bind, unbind, isBound, rebind) => {
     bind(Config).toConstantValue(ConfigFile.fromFile());
@@ -162,7 +163,8 @@ export const productionContainerModule = new ContainerModule((bind, unbind, isBo
         return { address: config.imageBuilderAddr };
     });
     bind(CachingImageBuilderClientProvider).toSelf().inSingletonScope();
-    bind(ImageBuilderClientProvider).toService(CachingImageBuilderClientProvider);
+    bind(WorkspaceClusterImagebuilderClientProvider).toSelf().inSingletonScope();
+    bind(ImageBuilderClientProvider).toService(WorkspaceClusterImagebuilderClientProvider);
     bind(ImageBuilderClientCallMetrics).toService(IClientCallMetrics);
 
     /* The binding order of the context parser does not configure preference/a working order. Each context parser must be able

components/server/src/workspace/gitpod-server-impl.ts

@@ -1562,7 +1562,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
 
             log.warn(logCtx, "imageBuild logs: fallback!");
             ctx.span?.setTag("workspace.imageBuild.logs.fallback", true);
-            await this.deprecatedDoWatchWorkspaceImageBuildLogs(ctx, logCtx, workspace);
+            await this.deprecatedDoWatchWorkspaceImageBuildLogs(ctx, logCtx, user, workspace);
             return;
         }
 
@@ -1611,6 +1611,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
     protected async deprecatedDoWatchWorkspaceImageBuildLogs(
         ctx: TraceContext,
         logCtx: LogContext,
+        user: User,
         workspace: Workspace,
     ) {
         if (!workspace.imageNameResolved) {
@@ -1619,7 +1620,11 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
         }
 
         try {
-            const imgbuilder = this.imageBuilderClientProvider.getDefault();
+            const imgbuilder = await this.imageBuilderClientProvider.getDefault(
+                user,
+                workspace,
+                {} as WorkspaceInstance,
+            );
             const req = new LogsRequest();
             req.setCensored(true);
             req.setBuildRef(workspace.imageNameResolved);

components/server/src/workspace/workspace-cluster-imagebuilder-client-provider.ts

@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import { Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol";
+import { IClientCallMetrics } from "@gitpod/gitpod-protocol/lib/messaging/client-call-metrics";
+import { defaultGRPCOptions } from "@gitpod/gitpod-protocol/lib/util/grpc";
+import {
+    ImageBuilderClient,
+    ImageBuilderClientCallMetrics,
+    ImageBuilderClientProvider,
+    PromisifiedImageBuilderClient,
+} from "@gitpod/image-builder/lib";
+import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider";
+import {
+    WorkspaceManagerClientProviderCompositeSource,
+    WorkspaceManagerClientProviderSource,
+} from "@gitpod/ws-manager/lib/client-provider-source";
+import { ExtendedUser } from "@gitpod/ws-manager/lib/constraints";
+import { inject, injectable, optional } from "inversify";
+
+@injectable()
+export class WorkspaceClusterImagebuilderClientProvider implements ImageBuilderClientProvider {
+    @inject(WorkspaceManagerClientProviderCompositeSource)
+    protected readonly source: WorkspaceManagerClientProviderSource;
+    @inject(WorkspaceManagerClientProvider) protected readonly clientProvider: WorkspaceManagerClientProvider;
+    @inject(ImageBuilderClientCallMetrics) @optional() protected readonly clientCallMetrics: IClientCallMetrics;
+
+    // gRPC connections can be used concurrently, even across services.
+    // Thus it makes sense to cache them rather than create a new connection for each request.
+    protected readonly connectionCache = new Map<string, ImageBuilderClient>();
+
+    async getDefault(
+        user: ExtendedUser,
+        workspace: Workspace,
+        instance: WorkspaceInstance,
+    ): Promise<PromisifiedImageBuilderClient> {
+        const clusters = await this.clientProvider.getStartClusterSets(user, workspace, instance);
+        for await (let cluster of clusters) {
+            const info = await this.source.getWorkspaceCluster(cluster.installation);
+            if (!info) {
+                continue;
+            }
+
+            var client = this.connectionCache.get(info.name);
+            if (!client) {
+                client = this.clientProvider.createConnection(ImageBuilderClient, info, defaultGRPCOptions);
+                this.connectionCache.set(info.name, client);
+            }
+            return new PromisifiedImageBuilderClient(client, []);
+        }
+
+        throw new Error("no image-builder available");
+    }
+}

components/server/src/workspace/workspace-starter.ts

@@ -249,7 +249,11 @@ export class WorkspaceStarter {
                 auth.setTotal(allowAll);
                 req.setAuth(auth);
 
-                const client = this.imagebuilderClientProvider.getDefault();
+                const client = await this.imagebuilderClientProvider.getDefault(
+                    user,
+                    workspace,
+                    {} as WorkspaceInstance,
+                );
                 const res = await client.resolveBaseImage({ span }, req);
                 workspace.imageSource = <WorkspaceImageSourceReference>{
                     baseImageResolved: res.getRef(),
@@ -902,7 +906,7 @@ export class WorkspaceStarter {
     ): Promise<boolean> {
         const span = TraceContext.startSpan("needsImageBuild", ctx);
         try {
-            const client = this.imagebuilderClientProvider.getDefault();
+            const client = await this.imagebuilderClientProvider.getDefault(user, workspace, instance);
             const { src, auth, disposable } = await this.prepareBuildRequest(
                 { span },
                 workspace,
@@ -942,7 +946,7 @@ export class WorkspaceStarter {
 
         try {
             // Start build...
-            const client = this.imagebuilderClientProvider.getDefault();
+            const client = await this.imagebuilderClientProvider.getDefault(user, workspace, instance);
             const { src, auth, disposable } = await this.prepareBuildRequest(
                 { span },
                 workspace,

components/ws-manager-api/typescript/src/client-provider.ts

@@ -92,14 +92,14 @@ export class WorkspaceManagerClientProvider implements Disposable {
         let client = this.connectionCache.get(name);
         if (!client) {
             const info = await getConnectionInfo();
-            client = this.createClient(info, grpcOptions);
+            client = client = this.createConnection(WorkspaceManagerClient, info, grpcOptions);
             this.connectionCache.set(name, client);
         } else if (client.getChannel().getConnectivityState(true) != grpc.connectivityState.READY) {
             client.close();
 
             console.warn(`Lost connection to workspace manager \"${name}\" - attempting to reestablish`);
             const info = await getConnectionInfo();
-            client = this.createClient(info, grpcOptions);
+            client = this.createConnection(WorkspaceManagerClient, info, grpcOptions);
             this.connectionCache.set(name, client);
         }
 
@@ -119,7 +119,12 @@ export class WorkspaceManagerClientProvider implements Disposable {
         return this.source.getAllWorkspaceClusters();
     }
 
-    public createClient(info: WorkspaceManagerConnectionInfo, grpcOptions?: object): WorkspaceManagerClient {
+    public createConnection<T extends grpc.Client>(creator: { new(address: string, credentials: grpc.ChannelCredentials, options?: grpc.ClientOptions): T }, info: WorkspaceManagerConnectionInfo, grpcOptions?: object): T {
+        const options: Partial<grpc.ClientOptions> = {
+            ...grpcOptions,
+            'grpc.ssl_target_name_override': "ws-manager",  // this makes sure we can call ws-manager with a URL different to "ws-manager"
+        };
+
         let credentials: grpc.ChannelCredentials;
         if (info.tls) {
             const rootCerts = Buffer.from(info.tls.ca, "base64");
@@ -131,11 +136,7 @@ export class WorkspaceManagerClientProvider implements Disposable {
             credentials = grpc.credentials.createInsecure();
         }
 
-        const options: Partial<grpc.ClientOptions> = {
-            ...grpcOptions,
-            'grpc.ssl_target_name_override': "ws-manager",  // this makes sure we can call ws-manager with a URL different to "ws-manager"
-        };
-        return new WorkspaceManagerClient(info.url, credentials, options);
+        return new creator(info.url, credentials, options);
     }
 
     public dispose() {

components/ws-manager-bridge/src/cluster-service-server.ts

@@ -34,7 +34,7 @@ import {
     UpdateResponse,
     AdmissionConstraint as GRPCAdmissionConstraint,
 } from "@gitpod/ws-manager-bridge-api/lib";
-import { GetWorkspacesRequest } from "@gitpod/ws-manager/lib";
+import { GetWorkspacesRequest, WorkspaceManagerClient } from "@gitpod/ws-manager/lib";
 import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider";
 import {
     WorkspaceManagerClientProviderCompositeSource,
@@ -151,7 +151,7 @@ export class ClusterService implements IClusterServiceServer {
 
                 // try to connect to validate the config. Throws an exception if it fails.
                 await new Promise<void>((resolve, reject) => {
-                    const c = this.clientProvider.createClient(newCluster);
+                    const c = this.clientProvider.createConnection(WorkspaceManagerClient, newCluster);
                     c.getWorkspaces(new GetWorkspacesRequest(), (err: any) => {
                         if (err) {
                             reject(