diff --git a/components/image-builder-api/typescript/src/sugar.ts b/components/image-builder-api/typescript/src/sugar.ts index 679fbfad85ff41..877832f38f0f60 100644 --- a/components/image-builder-api/typescript/src/sugar.ts +++ b/components/image-builder-api/typescript/src/sugar.ts @@ -15,13 +15,13 @@ import { BuildRequest, BuildResponse, BuildStatus, LogsRequest, LogsResponse, Re import { injectable, inject, optional } from 'inversify'; import * as grpc from "@grpc/grpc-js"; import { TextDecoder } from "util"; -import { ImageBuildLogInfo } from "@gitpod/gitpod-protocol"; +import { ImageBuildLogInfo, User, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol"; export const ImageBuilderClientProvider = Symbol("ImageBuilderClientProvider"); // ImageBuilderClientProvider caches image builder connections export interface ImageBuilderClientProvider { - getDefault(): PromisifiedImageBuilderClient + getDefault(user: User, workspace: Workspace, instance: WorkspaceInstance): Promise } function withTracing(ctx: TraceContext) { @@ -53,7 +53,7 @@ export class CachingImageBuilderClientProvider implements ImageBuilderClientProv // Thus it makes sense to cache them rather than create a new connection for each request. protected connectionCache: PromisifiedImageBuilderClient | undefined; - getDefault() { + async getDefault(user: User, workspace: Workspace, instance: WorkspaceInstance) { let interceptors: grpc.Interceptor[] = []; if (this.clientCallMetrics) { interceptors = [ createClientCallMetricsInterceptor(this.clientCallMetrics) ]; @@ -78,6 +78,18 @@ export class CachingImageBuilderClientProvider implements ImageBuilderClientProv return connection; } + promisify(c: ImageBuilderClient): PromisifiedImageBuilderClient { + let interceptors: grpc.Interceptor[] = []; + if (this.clientCallMetrics) { + interceptors = [ createClientCallMetricsInterceptor(this.clientCallMetrics) ]; + } + + return new PromisifiedImageBuilderClient( + new ImageBuilderClient(this.clientConfig.address, grpc.credentials.createInsecure()), + interceptors + ); + } + } // StagedBuildResponse captures the multi-stage nature (starting, running, done) of image builds. diff --git a/components/server/src/container-module.ts b/components/server/src/container-module.ts index e4ac066622fe52..5eeb2b0efe6306 100644 --- a/components/server/src/container-module.ts +++ b/components/server/src/container-module.ts @@ -97,6 +97,7 @@ import { LocalMessageBroker, LocalRabbitMQBackedMessageBroker } from "./messagin import { contentServiceBinder } from "@gitpod/content-service/lib/sugar"; import { ReferrerPrefixParser } from "./workspace/referrer-prefix-context-parser"; import { InstallationAdminTelemetryDataProvider } from "./installation-admin/telemetry-data-provider"; +import { WorkspaceClusterImagebuilderClientProvider } from "./workspace/workspace-cluster-imagebuilder-client-provider"; export const productionContainerModule = new ContainerModule((bind, unbind, isBound, rebind) => { bind(Config).toConstantValue(ConfigFile.fromFile()); @@ -159,7 +160,8 @@ export const productionContainerModule = new ContainerModule((bind, unbind, isBo return { address: config.imageBuilderAddr }; }); bind(CachingImageBuilderClientProvider).toSelf().inSingletonScope(); - bind(ImageBuilderClientProvider).toService(CachingImageBuilderClientProvider); + bind(WorkspaceClusterImagebuilderClientProvider).toSelf().inSingletonScope(); + bind(ImageBuilderClientProvider).toService(WorkspaceClusterImagebuilderClientProvider); bind(ImageBuilderClientCallMetrics).toService(IClientCallMetrics); /* The binding order of the context parser does not configure preference/a working order. Each context parser must be able diff --git a/components/server/src/workspace/gitpod-server-impl.ts b/components/server/src/workspace/gitpod-server-impl.ts index 7652e2893fe0c9..4d418054094302 100644 --- a/components/server/src/workspace/gitpod-server-impl.ts +++ b/components/server/src/workspace/gitpod-server-impl.ts @@ -108,7 +108,7 @@ import { RemotePageMessage, RemoteTrackMessage, } from "@gitpod/gitpod-protocol/lib/analytics"; -import { ImageBuilderClientProvider, LogsRequest } from "@gitpod/image-builder/lib"; +import { ImageBuilderClientProvider } from "@gitpod/image-builder/lib"; import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider"; import { ControlPortRequest, @@ -1550,11 +1550,11 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { // during roll-out this is our fall-back case. // Afterwards we might want to do some spinning-lock and re-check for a certain period (30s?) to give db-sync // a change to move the imageBuildLogInfo across the globe. - - log.warn(logCtx, "imageBuild logs: fallback!"); - ctx.span?.setTag("workspace.imageBuild.logs.fallback", true); - await this.deprecatedDoWatchWorkspaceImageBuildLogs(ctx, logCtx, workspace); - return; + log.error(logCtx, "cannot watch imagebuild logs for workspaceId: no image build info available"); + throw new ResponseError( + ErrorCodes.HEADLESS_LOG_NOT_YET_AVAILABLE, + "cannot watch imagebuild logs for workspaceId", + ); } const aborted = new Deferred(); @@ -1599,42 +1599,6 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { } } - protected async deprecatedDoWatchWorkspaceImageBuildLogs( - ctx: TraceContext, - logCtx: LogContext, - workspace: Workspace, - ) { - if (!workspace.imageNameResolved) { - log.debug(logCtx, `No imageNameResolved set for workspaceId, cannot watch logs.`); - return; - } - - try { - const imgbuilder = this.imageBuilderClientProvider.getDefault(); - const req = new LogsRequest(); - req.setCensored(true); - req.setBuildRef(workspace.imageNameResolved); - - let lineCount = 0; - await imgbuilder.logs(ctx, req, (data) => { - if (!this.client) { - return "stop"; - } - data = data.replace("\n", WorkspaceImageBuild.LogLine.DELIMITER); - lineCount += data.split(WorkspaceImageBuild.LogLine.DELIMITER_REGEX).length; - - this.client.onWorkspaceImageBuildLogs(undefined as any, { - text: data, - isDiff: true, - upToLine: lineCount, - }); - return "continue"; - }); - } catch (err) { - log.error(logCtx, `cannot watch logs for workspaceId`, err); - } - } - async getHeadlessLog(ctx: TraceContext, instanceId: string): Promise { traceAPIParams(ctx, { instanceId }); diff --git a/components/server/src/workspace/workspace-cluster-imagebuilder-client-provider.ts b/components/server/src/workspace/workspace-cluster-imagebuilder-client-provider.ts new file mode 100644 index 00000000000000..4f25f57863a444 --- /dev/null +++ b/components/server/src/workspace/workspace-cluster-imagebuilder-client-provider.ts @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import { Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol"; +import { IClientCallMetrics } from "@gitpod/gitpod-protocol/lib/messaging/client-call-metrics"; +import { defaultGRPCOptions } from "@gitpod/gitpod-protocol/lib/util/grpc"; +import { + ImageBuilderClient, + ImageBuilderClientCallMetrics, + ImageBuilderClientProvider, + PromisifiedImageBuilderClient, +} from "@gitpod/image-builder/lib"; +import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider"; +import { + WorkspaceManagerClientProviderCompositeSource, + WorkspaceManagerClientProviderSource, +} from "@gitpod/ws-manager/lib/client-provider-source"; +import { ExtendedUser } from "@gitpod/ws-manager/lib/constraints"; +import { inject, injectable, optional } from "inversify"; + +@injectable() +export class WorkspaceClusterImagebuilderClientProvider implements ImageBuilderClientProvider { + @inject(WorkspaceManagerClientProviderCompositeSource) + protected readonly source: WorkspaceManagerClientProviderSource; + @inject(WorkspaceManagerClientProvider) protected readonly clientProvider: WorkspaceManagerClientProvider; + @inject(ImageBuilderClientCallMetrics) @optional() protected readonly clientCallMetrics: IClientCallMetrics; + + // gRPC connections can be used concurrently, even across services. + // Thus it makes sense to cache them rather than create a new connection for each request. + protected readonly connectionCache = new Map(); + + async getDefault( + user: ExtendedUser, + workspace: Workspace, + instance: WorkspaceInstance, + ): Promise { + const clusters = await this.clientProvider.getStartClusterSets(user, workspace, instance); + for await (let cluster of clusters) { + const info = await this.source.getWorkspaceCluster(cluster.installation); + if (!info) { + continue; + } + + var client = this.connectionCache.get(info.name); + if (!client) { + client = this.clientProvider.createConnection(ImageBuilderClient, info, defaultGRPCOptions); + this.connectionCache.set(info.name, client); + } + return new PromisifiedImageBuilderClient(client, []); + } + + throw new Error("no image-builder available"); + } +} diff --git a/components/server/src/workspace/workspace-starter.ts b/components/server/src/workspace/workspace-starter.ts index dcca719f13fa10..eb10c2007d8ddf 100644 --- a/components/server/src/workspace/workspace-starter.ts +++ b/components/server/src/workspace/workspace-starter.ts @@ -178,7 +178,11 @@ export class WorkspaceStarter { auth.setTotal(allowAll); req.setAuth(auth); - const client = this.imagebuilderClientProvider.getDefault(); + const client = await this.imagebuilderClientProvider.getDefault( + user, + workspace, + {} as WorkspaceInstance, + ); const res = await client.resolveBaseImage({ span }, req); workspace.imageSource = { baseImageResolved: res.getRef(), @@ -814,7 +818,7 @@ export class WorkspaceStarter { ): Promise { const span = TraceContext.startSpan("needsImageBuild", ctx); try { - const client = this.imagebuilderClientProvider.getDefault(); + const client = await this.imagebuilderClientProvider.getDefault(user, workspace, instance); const { src, auth, disposable } = await this.prepareBuildRequest( { span }, workspace, @@ -854,7 +858,7 @@ export class WorkspaceStarter { try { // Start build... - const client = this.imagebuilderClientProvider.getDefault(); + const client = await this.imagebuilderClientProvider.getDefault(user, workspace, instance); const { src, auth, disposable } = await this.prepareBuildRequest( { span }, workspace, diff --git a/components/ws-manager-api/go/config/config.go b/components/ws-manager-api/go/config/config.go index ce8465bfb7f710..6346d64fcc3211 100644 --- a/components/ws-manager-api/go/config/config.go +++ b/components/ws-manager-api/go/config/config.go @@ -49,6 +49,9 @@ type ServiceConfiguration struct { } `json:"tls"` RateLimits map[string]grpc.RateLimit `json:"ratelimits"` } `json:"rpcServer"` + ImageBuilderProxy struct { + TargetAddr string `json:"targetAddr"` + } `json:"imageBuilderProxy"` PProf struct { Addr string `json:"addr"` diff --git a/components/ws-manager-api/typescript/src/client-provider.ts b/components/ws-manager-api/typescript/src/client-provider.ts index 423eb65f8aaa72..1229a5032217cf 100644 --- a/components/ws-manager-api/typescript/src/client-provider.ts +++ b/components/ws-manager-api/typescript/src/client-provider.ts @@ -92,14 +92,14 @@ export class WorkspaceManagerClientProvider implements Disposable { let client = this.connectionCache.get(name); if (!client) { const info = await getConnectionInfo(); - client = this.createClient(info, grpcOptions); + client = client = this.createConnection(WorkspaceManagerClient, info, grpcOptions); this.connectionCache.set(name, client); } else if (client.getChannel().getConnectivityState(true) != grpc.connectivityState.READY) { client.close(); console.warn(`Lost connection to workspace manager \"${name}\" - attempting to reestablish`); const info = await getConnectionInfo(); - client = this.createClient(info, grpcOptions); + client = this.createConnection(WorkspaceManagerClient, info, grpcOptions); this.connectionCache.set(name, client); } @@ -119,7 +119,12 @@ export class WorkspaceManagerClientProvider implements Disposable { return this.source.getAllWorkspaceClusters(); } - public createClient(info: WorkspaceManagerConnectionInfo, grpcOptions?: object): WorkspaceManagerClient { + public createConnection(creator: { new(address: string, credentials: grpc.ChannelCredentials, options?: grpc.ClientOptions): T }, info: WorkspaceManagerConnectionInfo, grpcOptions?: object): T { + const options: Partial = { + ...grpcOptions, + 'grpc.ssl_target_name_override': "ws-manager", // this makes sure we can call ws-manager with a URL different to "ws-manager" + }; + let credentials: grpc.ChannelCredentials; if (info.tls) { const rootCerts = Buffer.from(info.tls.ca, "base64"); @@ -131,11 +136,7 @@ export class WorkspaceManagerClientProvider implements Disposable { credentials = grpc.credentials.createInsecure(); } - const options: Partial = { - ...grpcOptions, - 'grpc.ssl_target_name_override': "ws-manager", // this makes sure we can call ws-manager with a URL different to "ws-manager" - }; - return new WorkspaceManagerClient(info.url, credentials, options); + return new creator(info.url, credentials, options); } public dispose() { diff --git a/components/ws-manager-bridge/src/cluster-service-server.ts b/components/ws-manager-bridge/src/cluster-service-server.ts index d08c40b68a8172..38101409554071 100644 --- a/components/ws-manager-bridge/src/cluster-service-server.ts +++ b/components/ws-manager-bridge/src/cluster-service-server.ts @@ -4,10 +4,20 @@ * See License-AGPL.txt in the project root for license information. */ -import { WorkspaceDB } from '@gitpod/gitpod-db/lib/workspace-db'; -import { Queue } from '@gitpod/gitpod-protocol'; -import { log } from '@gitpod/gitpod-protocol/lib/util/logging'; -import { WorkspaceCluster, WorkspaceClusterDB, WorkspaceClusterState, TLSConfig, AdmissionConstraint, AdmissionConstraintHasPermission, WorkspaceClusterWoTLS, AdmissionConstraintHasUserLevel, AdmissionConstraintHasMoreResources } from '@gitpod/gitpod-protocol/lib/workspace-cluster'; +import { WorkspaceDB } from "@gitpod/gitpod-db/lib/workspace-db"; +import { Queue } from "@gitpod/gitpod-protocol"; +import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; +import { + WorkspaceCluster, + WorkspaceClusterDB, + WorkspaceClusterState, + TLSConfig, + AdmissionConstraint, + AdmissionConstraintHasPermission, + WorkspaceClusterWoTLS, + AdmissionConstraintHasUserLevel, + AdmissionConstraintHasMoreResources, +} from "@gitpod/gitpod-protocol/lib/workspace-cluster"; import { ClusterServiceService, ClusterState, @@ -23,15 +33,18 @@ import { UpdateRequest, UpdateResponse, AdmissionConstraint as GRPCAdmissionConstraint, -} from '@gitpod/ws-manager-bridge-api/lib'; -import { GetWorkspacesRequest } from '@gitpod/ws-manager/lib'; -import { WorkspaceManagerClientProvider } from '@gitpod/ws-manager/lib/client-provider'; -import { WorkspaceManagerClientProviderCompositeSource, WorkspaceManagerClientProviderSource } from '@gitpod/ws-manager/lib/client-provider-source'; +} from "@gitpod/ws-manager-bridge-api/lib"; +import { GetWorkspacesRequest, WorkspaceManagerClient } from "@gitpod/ws-manager/lib"; +import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider"; +import { + WorkspaceManagerClientProviderCompositeSource, + WorkspaceManagerClientProviderSource, +} from "@gitpod/ws-manager/lib/client-provider-source"; import * as grpc from "@grpc/grpc-js"; -import { ServiceError as grpcServiceError } from '@grpc/grpc-js'; -import { inject, injectable } from 'inversify'; -import { BridgeController } from './bridge-controller'; -import { Configuration } from './config'; +import { ServiceError as grpcServiceError } from "@grpc/grpc-js"; +import { inject, injectable } from "inversify"; +import { BridgeController } from "./bridge-controller"; +import { Configuration } from "./config"; export interface ClusterServiceServerOptions { port: number; @@ -64,27 +77,33 @@ export class ClusterService implements IClusterServiceServer { // using a queue to make sure we do concurrency right protected readonly queue: Queue = new Queue(); - public register(call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData) { - log.info("requested clusters.register", getClientInfo(call)) + public register( + call: grpc.ServerUnaryCall, + callback: grpc.sendUnaryData, + ) { + log.info("requested clusters.register", getClientInfo(call)); this.queue.enqueue(async () => { try { // check if the name or URL are already registered/in use const req = call.request.toObject(); const clusterByNamePromise = this.clusterDB.findByName(req.name); - const clusterByUrlPromise = this.clusterDB.findFiltered({ url: req.url }) + const clusterByUrlPromise = this.clusterDB.findFiltered({ url: req.url }); - const [clusterByName, clusterByUrl] = await Promise.all([ - clusterByNamePromise, - clusterByUrlPromise - ]); + const [clusterByName, clusterByUrl] = await Promise.all([clusterByNamePromise, clusterByUrlPromise]); if (!!clusterByName) { - throw new GRPCError(grpc.status.ALREADY_EXISTS, `a WorkspaceCluster with name ${req.name} already exists in the DB`); + throw new GRPCError( + grpc.status.ALREADY_EXISTS, + `a WorkspaceCluster with name ${req.name} already exists in the DB`, + ); } if (!!clusterByUrl) { if (clusterByUrl.length > 0) { - throw new GRPCError(grpc.status.ALREADY_EXISTS, `a WorkspaceCluster with url ${req.url} already exists in the DB`); + throw new GRPCError( + grpc.status.ALREADY_EXISTS, + `a WorkspaceCluster with url ${req.url} already exists in the DB`, + ); } } @@ -111,10 +130,13 @@ export class ClusterService implements IClusterServiceServer { const tls: TLSConfig = { ca: req.tls.ca, crt: req.tls.crt, - key: req.tls.key + key: req.tls.key, }; - const admissionConstraints = call.request.getAdmissionConstraintsList().map(mapAdmissionConstraint).filter(c => !!c) as AdmissionConstraint[]; + const admissionConstraints = call.request + .getAdmissionConstraintsList() + .map(mapAdmissionConstraint) + .filter((c) => !!c) as AdmissionConstraint[]; const newCluster: WorkspaceCluster = { name: req.name, @@ -129,10 +151,15 @@ export class ClusterService implements IClusterServiceServer { // try to connect to validate the config. Throws an exception if it fails. await new Promise((resolve, reject) => { - const c = this.clientProvider.createClient(newCluster); + const c = this.clientProvider.createConnection(WorkspaceManagerClient, newCluster); c.getWorkspaces(new GetWorkspacesRequest(), (err: any) => { if (err) { - reject(new GRPCError(grpc.status.FAILED_PRECONDITION, `cannot reach ${req.url}: ${err.message}`)); + reject( + new GRPCError( + grpc.status.FAILED_PRECONDITION, + `cannot reach ${req.url}: ${err.message}`, + ), + ); } else { resolve(); } @@ -140,7 +167,7 @@ export class ClusterService implements IClusterServiceServer { }); await this.clusterDB.save(newCluster); - log.info({}, "cluster registered", {cluster: req.name}); + log.info({}, "cluster registered", { cluster: req.name }); this.triggerReconcile("register", req.name); callback(null, new RegisterResponse()); @@ -150,14 +177,20 @@ export class ClusterService implements IClusterServiceServer { }); } - public update(call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData) { - log.info("requested clusters.update", getClientInfo(call)) + public update( + call: grpc.ServerUnaryCall, + callback: grpc.sendUnaryData, + ) { + log.info("requested clusters.update", getClientInfo(call)); this.queue.enqueue(async () => { try { const req = call.request.toObject(); const cluster = await this.clusterDB.findByName(req.name); if (!cluster) { - throw new GRPCError(grpc.status.NOT_FOUND, `a WorkspaceCluster with name ${req.name} does not exist in the DB!`); + throw new GRPCError( + grpc.status.NOT_FOUND, + `a WorkspaceCluster with name ${req.name} does not exist in the DB!`, + ); } if (call.request.hasMaxScore()) { @@ -176,7 +209,7 @@ export class ClusterService implements IClusterServiceServer { if (mod.getAdd()) { cluster.admissionConstraints = (cluster.admissionConstraints || []).concat([c]); } else { - cluster.admissionConstraints = cluster.admissionConstraints?.filter(v => { + cluster.admissionConstraints = cluster.admissionConstraints?.filter((v) => { if (v.type !== c.type) { return true; } @@ -197,12 +230,12 @@ export class ClusterService implements IClusterServiceServer { return false; } return true; - }) + }); } } } await this.clusterDB.save(cluster); - log.info({}, "cluster updated", {cluster: req.name}); + log.info({}, "cluster updated", { cluster: req.name }); this.triggerReconcile("update", req.name); callback(null, new UpdateResponse()); @@ -212,21 +245,29 @@ export class ClusterService implements IClusterServiceServer { }); } - public deregister(call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData) { - log.info("requested clusters.deregister", getClientInfo(call)) + public deregister( + call: grpc.ServerUnaryCall, + callback: grpc.sendUnaryData, + ) { + log.info("requested clusters.deregister", getClientInfo(call)); this.queue.enqueue(async () => { try { const req = call.request.toObject(); const instances = await this.workspaceDB.findRegularRunningInstances(); - const relevantInstances = instances.filter(i => i.region === req.name); + const relevantInstances = instances.filter((i) => i.region === req.name); if (!req.force && relevantInstances.length > 0) { - log.info({}, "forced cluster deregistration even though there are still instances running", {cluster: req.name}); - throw new GRPCError(grpc.status.FAILED_PRECONDITION, `cluster is not empty (${relevantInstances.length} instances remaining)`); + log.info({}, "forced cluster deregistration even though there are still instances running", { + cluster: req.name, + }); + throw new GRPCError( + grpc.status.FAILED_PRECONDITION, + `cluster is not empty (${relevantInstances.length} instances remaining)`, + ); } await this.clusterDB.deleteByName(req.name); - log.info({}, "cluster deregistered", {cluster: req.name}); + log.info({}, "cluster deregistered", { cluster: req.name }); this.triggerReconcile("deregister", req.name); callback(null, new DeregisterResponse()); @@ -237,7 +278,7 @@ export class ClusterService implements IClusterServiceServer { } public list(call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData) { - log.info("requested clusters.list", getClientInfo(call)) + log.info("requested clusters.list", getClientInfo(call)); this.queue.enqueue(async () => { try { const response = new ListResponse(); @@ -271,8 +312,9 @@ export class ClusterService implements IClusterServiceServer { protected triggerReconcile(action: string, name: string) { const payload = { action, name }; log.info("reconcile: on request", payload); - this.bridgeController.runReconcileNow() - .catch(err => log.error("error during forced reconcile", err, payload)); + this.bridgeController + .runReconcileNow() + .catch((err) => log.error("error during forced reconcile", err, payload)); } } @@ -284,7 +326,7 @@ function convertToGRPC(ws: WorkspaceClusterWoTLS): ClusterStatus { clusterStatus.setScore(ws.score); clusterStatus.setMaxScore(ws.maxScore); clusterStatus.setGoverned(ws.govern); - ws.admissionConstraints?.forEach(c => { + ws.admissionConstraints?.forEach((c) => { const constraint = new GRPCAdmissionConstraint(); switch (c.type) { case "has-feature-preview": @@ -315,7 +357,7 @@ function mapAdmissionConstraint(c: GRPCAdmissionConstraint | undefined): Admissi } if (c.hasHasFeaturePreview()) { - return {type: "has-feature-preview"}; + return { type: "has-feature-preview" }; } if (c.hasHasPermission()) { const permission = c.getHasPermission()?.getPermission(); @@ -323,27 +365,31 @@ function mapAdmissionConstraint(c: GRPCAdmissionConstraint | undefined): Admissi return; } - return {type: "has-permission", permission}; + return { type: "has-permission", permission }; } if (c.hasHasUserLevel()) { const level = c.getHasUserLevel(); if (!level) { return; } - return {type: "has-user-level", level }; + return { type: "has-user-level", level }; } if (c.hasHasMoreResources()) { - return {type: "has-more-resources"}; + return { type: "has-more-resources" }; } return; } function mapPreferabilityToScore(p: Preferability): number | undefined { switch (p) { - case Preferability.PREFER: return 100; - case Preferability.NONE: return 50; - case Preferability.DONTSCHEDULE: return 0; - default: return undefined; + case Preferability.PREFER: + return 100; + case Preferability.NONE: + return 50; + case Preferability.DONTSCHEDULE: + return 0; + default: + return undefined; } } @@ -353,9 +399,12 @@ function mapCordoned(cordoned: boolean): WorkspaceClusterState { function mapClusterState(state: WorkspaceClusterState): ClusterState { switch (state) { - case 'available': return ClusterState.AVAILABLE; - case 'cordoned': return ClusterState.CORDONED; - case 'draining': return ClusterState.DRAINING; + case "available": + return ClusterState.AVAILABLE; + case "cordoned": + return ClusterState.CORDONED; + case "draining": + return ClusterState.DRAINING; } } @@ -366,19 +415,19 @@ function mapToGRPCError(err: any): any { return err; } -function getClientInfo(call: grpc.ServerUnaryCall) { - const clientNameHeader = call.metadata.get("client-name") - const userAgentHeader = call.metadata.get("user-agent") +function getClientInfo(call: grpc.ServerUnaryCall) { + const clientNameHeader = call.metadata.get("client-name"); + const userAgentHeader = call.metadata.get("user-agent"); - let [clientName, userAgent] = ["", ""] + let [clientName, userAgent] = ["", ""]; if (clientNameHeader.length != 0) { - clientName = clientNameHeader[0].toString() + clientName = clientNameHeader[0].toString(); } if (userAgentHeader.length != 0) { - userAgent = userAgentHeader[0].toString() + userAgent = userAgentHeader[0].toString(); } const clientIP = call.getPeer(); - return {clientIP, clientName, userAgent} + return { clientIP, clientName, userAgent }; } // "grpc" does not allow additional methods on it's "ServiceServer"s so we have an additional wrapper here @@ -396,7 +445,7 @@ export class ClusterServiceServer { // Default value for maxSessionMemory is 10 which is low for this gRPC server // See https://nodejs.org/api/http2.html#http2_http2_connect_authority_options_listener. const server = new grpc.Server({ - 'grpc-node.max_session_memory': 50 + "grpc-node.max_session_memory": 50, }); // @ts-ignore server.addService(ClusterServiceService, this.service); @@ -422,17 +471,14 @@ export class ClusterServiceServer { this.server = undefined; } } - } class GRPCError extends Error implements Partial { - public name = 'ServiceError'; + public name = "ServiceError"; details: string; - constructor( - public readonly status: grpc.status, - err: any) { + constructor(public readonly status: grpc.status, err: any) { super(GRPCError.errToMessage(err)); this.details = this.message; @@ -447,8 +493,6 @@ class GRPCError extends Error implements Partial { } static isGRPCError(obj: any): obj is GRPCError { - return obj !== undefined - && typeof obj === "object" - && "status" in obj; + return obj !== undefined && typeof obj === "object" && "status" in obj; } } diff --git a/components/ws-manager/cmd/run.go b/components/ws-manager/cmd/run.go index 919fa85538b0cd..a3dbd0b59f1a5a 100644 --- a/components/ws-manager/cmd/run.go +++ b/components/ws-manager/cmd/run.go @@ -7,6 +7,7 @@ package cmd import ( "context" "net" + "strings" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -14,9 +15,13 @@ import ( "github.com/bombsimon/logrusr/v2" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/mwitkow/grpc-proxy/proxy" "github.com/spf13/cobra" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" @@ -138,6 +143,8 @@ var runCmd = &cobra.Command{ log.Warn("no TLS configured - gRPC server will be unsecured") } + grpcOpts = append(grpcOpts, grpc.UnknownServiceHandler(proxy.TransparentHandler(imagebuilderDirector(cfg.ImageBuilderProxy.TargetAddr)))) + grpcServer := grpc.NewServer(grpcOpts...) defer grpcServer.Stop() grpc_prometheus.Register(grpcServer) @@ -200,3 +207,23 @@ func init() { var ( scheme = runtime.NewScheme() ) + +func imagebuilderDirector(targetAddr string) proxy.StreamDirector { + if targetAddr == "" { + return func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) { + return ctx, nil, status.Error(codes.Unimplemented, "Unknown method") + } + } + + return func(ctx context.Context, fullMethodName string) (outCtx context.Context, conn *grpc.ClientConn, err error) { + md, _ := metadata.FromIncomingContext(ctx) + outCtx = metadata.NewOutgoingContext(ctx, md.Copy()) + + if strings.HasPrefix(fullMethodName, "/builder.") { + conn, err = grpc.DialContext(ctx, targetAddr, grpc.WithInsecure()) + return + } + + return outCtx, nil, status.Error(codes.Unimplemented, "Unknown method") + } +} diff --git a/components/ws-manager/go.mod b/components/ws-manager/go.mod index 266e09a280b605..ff9d2c042f7108 100644 --- a/components/ws-manager/go.mod +++ b/components/ws-manager/go.mod @@ -44,6 +44,7 @@ require ( github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/BurntSushi/toml v0.3.1 // indirect github.com/Microsoft/hcsshim v0.8.17 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -81,6 +82,7 @@ require ( github.com/moby/sys/mountinfo v0.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mwitkow/grpc-proxy v0.0.0-20220126150247-db34e7bfee32 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -111,6 +113,7 @@ require ( gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + honnef.co/go/tools v0.1.3 // indirect k8s.io/apiextensions-apiserver v0.23.4 // indirect k8s.io/component-base v0.23.4 // indirect k8s.io/klog/v2 v2.30.0 // indirect diff --git a/components/ws-manager/go.sum b/components/ws-manager/go.sum index 9145e7ddc8a088..c530721f64d643 100644 --- a/components/ws-manager/go.sum +++ b/components/ws-manager/go.sum @@ -60,6 +60,7 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgRRqzCuPshRkQ7I= @@ -590,6 +591,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/grpc-proxy v0.0.0-20220126150247-db34e7bfee32 h1:CC9KzU7WPrK6DTppkUGiwmttoHCNwOLT7Z+stp1eIpU= +github.com/mwitkow/grpc-proxy v0.0.0-20220126150247-db34e7bfee32/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -938,6 +941,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1050,6 +1054,7 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1232,6 +1237,7 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210401141331-865547bb08e2/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210420162539-3c870d7478d2/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= @@ -1334,6 +1340,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8s.io/api v0.23.4 h1:85gnfXQOWbJa1SiWGpE9EEtHs0UVvDyIsSMpEtl2D4E= k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI= k8s.io/apiextensions-apiserver v0.23.4 h1:AFDUEu/yEf0YnuZhqhIFhPLPhhcQQVuR1u3WCh0rveU= diff --git a/install/installer/pkg/common/constants.go b/install/installer/pkg/common/constants.go index b8e0a4bc71a3df..6b59730c1c8e6e 100644 --- a/install/installer/pkg/common/constants.go +++ b/install/installer/pkg/common/constants.go @@ -43,6 +43,8 @@ const ( WSManagerComponent = "ws-manager" WSManagerBridgeComponent = "ws-manager-bridge" WSProxyComponent = "ws-proxy" + ImageBuilderComponent = "image-builder-mk3" + ImageBuilderRPCPort = 8080 AnnotationConfigChecksum = "gitpod.io/checksum_config" ) diff --git a/install/installer/pkg/components/components-webapp/components.go b/install/installer/pkg/components/components-webapp/components.go index b03bf902385bec..b66bb9e3dd3096 100644 --- a/install/installer/pkg/components/components-webapp/components.go +++ b/install/installer/pkg/components/components-webapp/components.go @@ -10,7 +10,6 @@ import ( "github.com/gitpod-io/gitpod/installer/pkg/components/dashboard" "github.com/gitpod-io/gitpod/installer/pkg/components/database" ide_proxy "github.com/gitpod-io/gitpod/installer/pkg/components/ide-proxy" - imagebuildermk3 "github.com/gitpod-io/gitpod/installer/pkg/components/image-builder-mk3" "github.com/gitpod-io/gitpod/installer/pkg/components/migrations" "github.com/gitpod-io/gitpod/installer/pkg/components/minio" openvsxproxy "github.com/gitpod-io/gitpod/installer/pkg/components/openvsx-proxy" @@ -25,7 +24,6 @@ var Objects = common.CompositeRenderFunc( dashboard.Objects, database.Objects, ide_proxy.Objects, - imagebuildermk3.Objects, migrations.Objects, minio.Objects, openvsxproxy.Objects, diff --git a/install/installer/pkg/components/components-workspace/components.go b/install/installer/pkg/components/components-workspace/components.go index f425440ee54769..b506a741fb3a05 100644 --- a/install/installer/pkg/components/components-workspace/components.go +++ b/install/installer/pkg/components/components-workspace/components.go @@ -8,6 +8,7 @@ import ( "github.com/gitpod-io/gitpod/installer/pkg/common" agentsmith "github.com/gitpod-io/gitpod/installer/pkg/components/agent-smith" "github.com/gitpod-io/gitpod/installer/pkg/components/blobserve" + imagebuildermk3 "github.com/gitpod-io/gitpod/installer/pkg/components/image-builder-mk3" registryfacade "github.com/gitpod-io/gitpod/installer/pkg/components/registry-facade" "github.com/gitpod-io/gitpod/installer/pkg/components/workspace" wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon" @@ -23,6 +24,7 @@ var Objects = common.CompositeRenderFunc( wsdaemon.Objects, wsmanager.Objects, wsproxy.Objects, + imagebuildermk3.Objects, ) var Helm = common.CompositeHelmFunc() diff --git a/install/installer/pkg/components/image-builder-mk3/constants.go b/install/installer/pkg/components/image-builder-mk3/constants.go index c58be07d358ff9..a1af0f08f3e59c 100644 --- a/install/installer/pkg/components/image-builder-mk3/constants.go +++ b/install/installer/pkg/components/image-builder-mk3/constants.go @@ -4,11 +4,13 @@ package image_builder_mk3 +import "github.com/gitpod-io/gitpod/installer/pkg/common" + const ( PullSecretFile = "/config/pull-secret.json" BuilderImage = "image-builder-mk3/bob" - Component = "image-builder-mk3" - RPCPort = 8080 + Component = common.ImageBuilderComponent + RPCPort = common.ImageBuilderRPCPort RPCPortName = "service" PProfPort = 6060 PrometheusPort = 9500 diff --git a/install/installer/pkg/components/image-builder-mk3/networkpolicy.go b/install/installer/pkg/components/image-builder-mk3/networkpolicy.go index 798e27ef5cc12c..2346818103cece 100644 --- a/install/installer/pkg/components/image-builder-mk3/networkpolicy.go +++ b/install/installer/pkg/components/image-builder-mk3/networkpolicy.go @@ -27,13 +27,26 @@ func networkpolicy(ctx *common.RenderContext) ([]runtime.Object, error) { Spec: networkingv1.NetworkPolicySpec{ PodSelector: metav1.LabelSelector{MatchLabels: labels}, PolicyTypes: []networkingv1.PolicyType{"Ingress"}, - Ingress: []networkingv1.NetworkPolicyIngressRule{{ - From: []networkingv1.NetworkPolicyPeer{{ - PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ - "component": server.Component, - }}, - }}, - }}, + Ingress: []networkingv1.NetworkPolicyIngressRule{ + { + From: []networkingv1.NetworkPolicyPeer{ + { + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": server.Component, + }, + }, + }, + { + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": common.WSManagerComponent, + }, + }, + }, + }, + }, + }, }, }, }, nil diff --git a/install/installer/pkg/components/ws-manager/configmap.go b/install/installer/pkg/components/ws-manager/configmap.go index a7b28551471b2f..276ca163fa6c3e 100644 --- a/install/installer/pkg/components/ws-manager/configmap.go +++ b/install/installer/pkg/components/ws-manager/configmap.go @@ -124,6 +124,11 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { }, RateLimits: map[string]grpc.RateLimit{}, // todo(sje) add values }, + ImageBuilderProxy: struct { + TargetAddr string "json:\"targetAddr\"" + }{ + TargetAddr: fmt.Sprintf("%s.%s.svc.cluster.local:%d", common.ImageBuilderComponent, ctx.Namespace, common.ImageBuilderRPCPort), + }, PProf: struct { Addr string `json:"addr"` }{Addr: "localhost:6060"},