diff --git a/components/gitpod-protocol/src/experiments/types.ts b/components/gitpod-protocol/src/experiments/types.ts index 2092baabea220b..943bdeace2cd24 100644 --- a/components/gitpod-protocol/src/experiments/types.ts +++ b/components/gitpod-protocol/src/experiments/types.ts @@ -6,6 +6,8 @@ import { Team } from "../teams-projects-protocol"; +export const Client = Symbol("Client"); + // Attributes define attributes which can be used to segment audiences. // Set the attributes which you want to use to group audiences into. export interface Attributes { diff --git a/components/gitpod-protocol/src/workspace-cluster.ts b/components/gitpod-protocol/src/workspace-cluster.ts index 2ab034a52f19c5..07c94ea5482f27 100644 --- a/components/gitpod-protocol/src/workspace-cluster.ts +++ b/components/gitpod-protocol/src/workspace-cluster.ts @@ -57,11 +57,13 @@ export type AdmissionConstraint = | AdmissionConstraintFeaturePreview | AdmissionConstraintHasPermission | AdmissionConstraintHasUserLevel - | AdmissionConstraintHasMoreResources; + | AdmissionConstraintHasMoreResources + | AdmissionConstraintHasClass; export type AdmissionConstraintFeaturePreview = { type: "has-feature-preview" }; export type AdmissionConstraintHasPermission = { type: "has-permission"; permission: PermissionName }; export type AdmissionConstraintHasUserLevel = { type: "has-user-level"; level: string }; export type AdmissionConstraintHasMoreResources = { type: "has-more-resources" }; +export type AdmissionConstraintHasClass = { type: "has-class"; id: string; displayName: string }; export namespace AdmissionConstraint { export function is(o: any): o is AdmissionConstraint { diff --git a/components/ws-manager-bridge/debug.sh b/components/ws-manager-bridge/debug.sh new file mode 100755 index 00000000000000..3351004d7187e8 --- /dev/null +++ b/components/ws-manager-bridge/debug.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the GNU Affero General Public License (AGPL). +# See License-AGPL.txt in the project root for license information. + +set -Eeuo pipefail + +source /workspace/gitpod/scripts/ws-deploy.sh deployment ws-manager-bridge diff --git a/components/ws-manager-bridge/src/cluster-service-server.ts b/components/ws-manager-bridge/src/cluster-service-server.ts index c9701e9c8688f9..6b23691fa5bda0 100644 --- a/components/ws-manager-bridge/src/cluster-service-server.ts +++ b/components/ws-manager-bridge/src/cluster-service-server.ts @@ -41,10 +41,12 @@ import { WorkspaceManagerClientProviderSource, } from "@gitpod/ws-manager/lib/client-provider-source"; import * as grpc from "@grpc/grpc-js"; -import { ServiceError as grpcServiceError } from "@grpc/grpc-js"; import { inject, injectable } from "inversify"; import { BridgeController } from "./bridge-controller"; +import { getSupportedWorkspaceClasses } from "./cluster-sync-service"; import { Configuration } from "./config"; +import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server"; +import { GRPCError } from "./rpc"; export interface ClusterServiceServerOptions { port: number; @@ -146,25 +148,34 @@ export class ClusterService implements IClusterServiceServer { maxScore: 100, govern, tls, - admissionConstraints, }; - // try to connect to validate the config. Throws an exception if it fails. - await new Promise((resolve, reject) => { - const c = this.clientProvider.createClient(newCluster); - c.getWorkspaces(new GetWorkspacesRequest(), (err: any) => { - if (err) { - reject( - new GRPCError( - grpc.status.FAILED_PRECONDITION, - `cannot reach ${req.url}: ${err.message}`, - ), - ); - } else { - resolve(); - } + const enabled = await getExperimentsClientForBackend().getValueAsync( + "workspace_classes_backend", + false, + {}, + ); + if (enabled) { + let classConstraints = await getSupportedWorkspaceClasses(this.clientProvider, newCluster, false); + newCluster.admissionConstraints = admissionConstraints.concat(classConstraints); + } else { + // try to connect to validate the config. Throws an exception if it fails. + await new Promise((resolve, reject) => { + const c = this.clientProvider.createClient(newCluster); + c.getWorkspaces(new GetWorkspacesRequest(), (err: any) => { + if (err) { + reject( + new GRPCError( + grpc.status.FAILED_PRECONDITION, + `cannot reach ${req.url}: ${err.message}`, + ), + ); + } else { + resolve(); + } + }); }); - }); + } await this.clusterDB.save(newCluster); log.info({}, "cluster registered", { cluster: req.name }); @@ -472,27 +483,3 @@ export class ClusterServiceServer { } } } - -class GRPCError extends Error implements Partial { - public name = "ServiceError"; - - details: string; - - constructor(public readonly status: grpc.status, err: any) { - super(GRPCError.errToMessage(err)); - - this.details = this.message; - } - - static errToMessage(err: any): string | undefined { - if (typeof err === "string") { - return err; - } else if (typeof err === "object") { - return err.message; - } - } - - static isGRPCError(obj: any): obj is GRPCError { - return obj !== undefined && typeof obj === "object" && "status" in obj; - } -} diff --git a/components/ws-manager-bridge/src/cluster-sync-service.ts b/components/ws-manager-bridge/src/cluster-sync-service.ts new file mode 100644 index 00000000000000..dbedbafd202829 --- /dev/null +++ b/components/ws-manager-bridge/src/cluster-sync-service.ts @@ -0,0 +1,95 @@ +/** + * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; +import { WorkspaceCluster, WorkspaceClusterDB } from "@gitpod/gitpod-protocol/lib/workspace-cluster"; +import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider"; +import { inject, injectable } from "inversify"; +import { Configuration } from "./config"; +import { Client } from "@gitpod/gitpod-protocol/lib/experiments/types"; +import { DescribeClusterRequest, DescribeClusterResponse, WorkspaceClass } from "@gitpod/ws-manager/lib"; +import { AdmissionConstraintHasClass } from "@gitpod/gitpod-protocol/src/workspace-cluster"; +import { GRPCError } from "./rpc"; +import * as grpc from "@grpc/grpc-js"; +import { defaultGRPCOptions } from "@gitpod/gitpod-protocol/lib/util/grpc"; + +@injectable() +export class ClusterSyncService { + @inject(Configuration) + protected readonly config: Configuration; + + @inject(WorkspaceClusterDB) + protected readonly clusterDB: WorkspaceClusterDB; + + @inject(WorkspaceManagerClientProvider) + protected readonly clientProvider: WorkspaceManagerClientProvider; + + @inject(Client) + protected readonly featureClient: Client; + + protected timer: NodeJS.Timer; + + public start() { + this.timer = setInterval(() => this.reconcile(), this.config.clusterSyncIntervalSeconds * 1000); + } + + private async reconcile() { + const enabled = await this.featureClient.getValueAsync("workspace_classes_backend", false, {}); + if (!enabled) { + return; + } + + log.debug("reconciling workspace classes..."); + let allClusters = await this.clusterDB.findFiltered({}); + for (const cluster of allClusters) { + try { + let supportedClasses = await getSupportedWorkspaceClasses(this.clientProvider, cluster, true); + let existingOtherConstraints = cluster.admissionConstraints?.filter((c) => c.type !== "has-class"); + cluster.admissionConstraints = existingOtherConstraints?.concat(supportedClasses); + await this.clusterDB.save(cluster); + } catch (err) { + log.error("failed to reconcile workspace classes for cluster", err, { cluster: cluster.name }); + } + } + log.debug("done reconciling workspace classes"); + } + + public stop() { + clearInterval(this.timer); + } +} + +export async function getSupportedWorkspaceClasses( + clientProvider: WorkspaceManagerClientProvider, + cluster: WorkspaceCluster, + useCache: boolean, +) { + let constraints = await new Promise(async (resolve, reject) => { + const grpcOptions: grpc.ClientOptions = { + ...defaultGRPCOptions, + }; + let client = useCache + ? await ( + await clientProvider.get(cluster.name, grpcOptions) + ).client + : clientProvider.createClient(cluster, grpcOptions); + + client.describeCluster(new DescribeClusterRequest(), (err: any, resp: DescribeClusterResponse) => { + if (err) { + reject(new GRPCError(grpc.status.FAILED_PRECONDITION, `cannot reach ${cluster.url}: ${err.message}`)); + } else { + let classes = resp.getWorkspaceclassesList().map((cl) => mapWorkspaceClass(cl)); + resolve(classes); + } + }); + }); + + return constraints; +} + +function mapWorkspaceClass(c: WorkspaceClass): AdmissionConstraintHasClass { + return { type: "has-class", id: c.getId(), displayName: c.getDisplayname() }; +} diff --git a/components/ws-manager-bridge/src/config.ts b/components/ws-manager-bridge/src/config.ts index 4d33d0f92a5ab4..2ab8adc2d12e78 100644 --- a/components/ws-manager-bridge/src/config.ts +++ b/components/ws-manager-bridge/src/config.ts @@ -35,4 +35,7 @@ export interface Configuration { // emulatePreparingIntervalSeconds configures how often we check for Workspaces in phase "preparing" for clusters we do not govern emulatePreparingIntervalSeconds: number; + + // clusterSyncIntervalSeconds configures how often we sync workspace cluster information + clusterSyncIntervalSeconds: number; } diff --git a/components/ws-manager-bridge/src/container-module.ts b/components/ws-manager-bridge/src/container-module.ts index 75d2ed4c803603..8a1aac78bb859d 100644 --- a/components/ws-manager-bridge/src/container-module.ts +++ b/components/ws-manager-bridge/src/container-module.ts @@ -35,6 +35,9 @@ import { PreparingUpdateEmulator, PreparingUpdateEmulatorFactory } from "./prepa import { PrebuildStateMapper } from "./prebuild-state-mapper"; import { PrebuildUpdater, PrebuildUpdaterNoOp } from "./prebuild-updater"; import { DebugApp } from "@gitpod/gitpod-protocol/lib/util/debug-app"; +import { Client } from "@gitpod/gitpod-protocol/lib/experiments/types"; +import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server"; +import { ClusterSyncService } from "./cluster-sync-service"; export const containerModule = new ContainerModule((bind) => { bind(MessagebusConfiguration).toSelf().inSingletonScope(); @@ -57,6 +60,7 @@ export const containerModule = new ContainerModule((bind) => { bind(ClusterServiceServer).toSelf().inSingletonScope(); bind(ClusterService).toSelf().inRequestScope(); + bind(ClusterSyncService).toSelf().inSingletonScope(); bind(TracingManager).toSelf().inSingletonScope(); @@ -85,4 +89,6 @@ export const containerModule = new ContainerModule((bind) => { bind(PrebuildUpdater).to(PrebuildUpdaterNoOp).inSingletonScope(); bind(DebugApp).toSelf().inSingletonScope(); + + bind(Client).toDynamicValue(getExperimentsClientForBackend).inSingletonScope(); }); diff --git a/components/ws-manager-bridge/src/main.ts b/components/ws-manager-bridge/src/main.ts index 5ff299bb228fb6..7fbf4b8c4b5ef3 100644 --- a/components/ws-manager-bridge/src/main.ts +++ b/components/ws-manager-bridge/src/main.ts @@ -14,6 +14,7 @@ import { TypeORM } from "@gitpod/gitpod-db/lib/typeorm/typeorm"; import { TracingManager } from "@gitpod/gitpod-protocol/lib/util/tracing"; import { ClusterServiceServer } from "./cluster-service-server"; import { BridgeController } from "./bridge-controller"; +import { ClusterSyncService } from "./cluster-sync-service"; log.enableJSONLogging("ws-manager-bridge", undefined, LogrusLogLevel.getFromEnv()); @@ -48,6 +49,9 @@ export const start = async (container: Container) => { const clusterServiceServer = container.get(ClusterServiceServer); await clusterServiceServer.start(); + const clusterSyncService = container.get(ClusterSyncService); + clusterSyncService.start(); + process.on("SIGTERM", async () => { log.info("SIGTERM received, stopping"); bridgeController.dispose(); diff --git a/components/ws-manager-bridge/src/rpc.ts b/components/ws-manager-bridge/src/rpc.ts new file mode 100644 index 00000000000000..5aff8ca623d22d --- /dev/null +++ b/components/ws-manager-bridge/src/rpc.ts @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import * as grpc from "@grpc/grpc-js"; +import { ServiceError as grpcServiceError } from "@grpc/grpc-js"; + +export class GRPCError extends Error implements Partial { + public name = "ServiceError"; + + details: string; + + constructor(public readonly status: grpc.status, err: any) { + super(GRPCError.errToMessage(err)); + + this.details = this.message; + } + + static errToMessage(err: any): string | undefined { + if (typeof err === "string") { + return err; + } else if (typeof err === "object") { + return err.message; + } + } + + static isGRPCError(obj: any): obj is GRPCError { + return obj !== undefined && typeof obj === "object" && "status" in obj; + } +} diff --git a/install/installer/pkg/common/common.go b/install/installer/pkg/common/common.go index f18de0f63a581b..9fa34801d7baa8 100644 --- a/install/installer/pkg/common/common.go +++ b/install/installer/pkg/common/common.go @@ -263,6 +263,27 @@ func DatabaseEnv(cfg *config.Config) (res []corev1.EnvVar) { return envvars } +func ConfigcatEnv(ctx *RenderContext) []corev1.EnvVar { + var sdkKey string + _ = ctx.WithExperimental(func(cfg *experimental.Config) error { + if cfg.WebApp != nil && cfg.WebApp.ConfigcatKey != "" { + sdkKey = cfg.WebApp.ConfigcatKey + } + return nil + }) + + if sdkKey == "" { + return nil + } + + return []corev1.EnvVar{ + { + Name: "CONFIGCAT_SDK_KEY", + Value: sdkKey, + }, + } +} + func DatabaseWaiterContainer(ctx *RenderContext) *corev1.Container { return &corev1.Container{ Name: "database-waiter", diff --git a/install/installer/pkg/components/server/deployment.go b/install/installer/pkg/components/server/deployment.go index f238f25a434b4e..26b904128233b0 100644 --- a/install/installer/pkg/components/server/deployment.go +++ b/install/installer/pkg/components/server/deployment.go @@ -77,7 +77,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { common.WebappTracingEnv(ctx), common.AnalyticsEnv(&ctx.Config), common.MessageBusEnv(&ctx.Config), - configcatEnv(ctx), + common.ConfigcatEnv(ctx), []corev1.EnvVar{ { Name: "CONFIG_PATH", @@ -402,24 +402,3 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { }, }, nil } - -func configcatEnv(ctx *common.RenderContext) []corev1.EnvVar { - var sdkKey string - _ = ctx.WithExperimental(func(cfg *experimental.Config) error { - if cfg.WebApp != nil && cfg.WebApp.ConfigcatKey != "" { - sdkKey = cfg.WebApp.ConfigcatKey - } - return nil - }) - - if sdkKey == "" { - return nil - } - - return []corev1.EnvVar{ - { - Name: "CONFIGCAT_SDK_KEY", - Value: sdkKey, - }, - } -} diff --git a/install/installer/pkg/components/ws-manager-bridge/configmap.go b/install/installer/pkg/components/ws-manager-bridge/configmap.go index 3d2ff89143be99..4963c3f3ad5b87 100644 --- a/install/installer/pkg/components/ws-manager-bridge/configmap.go +++ b/install/installer/pkg/components/ws-manager-bridge/configmap.go @@ -31,6 +31,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { }, EmulatePreparingIntervalSeconds: 10, StaticBridges: WSManagerList(ctx), + ClusterSyncIntervalSeconds: 60, } fc, err := common.ToJSONString(wsmbcfg) diff --git a/install/installer/pkg/components/ws-manager-bridge/deployment.go b/install/installer/pkg/components/ws-manager-bridge/deployment.go index b6a3882cc5b982..7fd3d158d5c4fe 100644 --- a/install/installer/pkg/components/ws-manager-bridge/deployment.go +++ b/install/installer/pkg/components/ws-manager-bridge/deployment.go @@ -130,6 +130,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { common.AnalyticsEnv(&ctx.Config), common.MessageBusEnv(&ctx.Config), common.DatabaseEnv(&ctx.Config), + common.ConfigcatEnv(ctx), []corev1.EnvVar{{ Name: "WSMAN_BRIDGE_CONFIGPATH", Value: "/config/ws-manager-bridge.json", diff --git a/install/installer/pkg/components/ws-manager-bridge/types.go b/install/installer/pkg/components/ws-manager-bridge/types.go index 2d6212e5b04204..2f49bc4a6b785b 100644 --- a/install/installer/pkg/components/ws-manager-bridge/types.go +++ b/install/installer/pkg/components/ws-manager-bridge/types.go @@ -14,6 +14,7 @@ type Configuration struct { ControllerMaxDisconnectSeconds int32 `json:"controllerMaxDisconnectSeconds"` EmulatePreparingIntervalSeconds int32 `json:"emulatePreparingIntervalSeconds"` Timeouts Timeouts `json:"timeouts"` + ClusterSyncIntervalSeconds int32 `json:"clusterSyncIntervalSeconds"` } type ClusterService struct {