[registry-facade] Support Redis sentinel clients

Author: csweichel

components/registry-facade-api/go/config/config.go

@@ -31,6 +31,12 @@ func GetConfig(fn string) (*ServiceConfig, error) {
 		return nil, err
 	}
 
+	if cfg.Registry.IPFSCache != nil {
+		rd := cfg.Registry.IPFSCache.Redis
+		rd.Password = os.Getenv("REDIS_PASSWORD")
+		cfg.Registry.IPFSCache.Redis = rd
+	}
+
 	return &cfg, nil
 }
 
@@ -60,9 +66,18 @@ type Config struct {
 }
 
 type IPFSCacheConfig struct {
-	Enabled   bool   `json:"enabled"`
-	RedisAddr string `json:"redis"`
-	IPFSAddr  string `json:"ipfs"`
+	Enabled  bool        `json:"enabled"`
+	Redis    RedisConfig `json:"redis"`
+	IPFSAddr string      `json:"ipfs"`
+}
+
+type RedisConfig struct {
+	SingleHostAddress string `json:"singleHostAddr,omitempty"`
+
+	MasterName    string   `json:"masterName,omitempty"`
+	SentinelAddrs []string `json:"sentinelAddrs,omitempty"`
+	Username      string   `json:"username,omitempty"`
+	Password      string   `json:"-" env:"REDIS_PASSWORD"`
 }
 
 // StaticLayerCfg configure statically added layer

components/registry-facade/example-config.json

@@ -13,7 +13,9 @@
     "fixedSpecFN": "example-spec.json",
     "ipfs": {
       "enabled": true,
-      "redis": "localhost:6379",
+      "redis": {
+        "singleHostAddr": "localhost:6379"
+      },
       "ipfs": "/ip4/127.0.0.1/tcp/5001"
     }
   },

components/registry-facade/pkg/registry/registry.go

@@ -205,9 +205,11 @@ func NewRegistry(cfg config.Config, newResolver ResolverProvider, reg prometheus
 		if err != nil {
 			return nil, xerrors.Errorf("cannot connect to IPFS: %w", err)
 		}
-		rdc := redis.NewClient(&redis.Options{
-			Addr: cfg.IPFSCache.RedisAddr,
-		})
+		rdc, err := getRedisClient(cfg.IPFSCache.Redis)
+		if err != nil {
+			return nil, xerrors.Errorf("cannot connect to Redis: %w", err)
+		}
+
 		ipfs = &IPFSStore{
 			Redis: rdc,
 			IPFS:  core,
@@ -229,6 +231,30 @@ func NewRegistry(cfg config.Config, newResolver ResolverProvider, reg prometheus
 	}, nil
 }
 
+func getRedisClient(cfg config.RedisConfig) (*redis.Client, error) {
+	if cfg.SingleHostAddress != "" {
+		log.WithField("addr", cfg.SingleHostAddress).WithField("username", cfg.Username).Info("connecting to single Redis host")
+		return redis.NewClient(&redis.Options{
+			Addr:     cfg.SingleHostAddress,
+			Username: cfg.Username,
+			Password: cfg.Password,
+		}), nil
+	}
+
+	if cfg.MasterName == "" {
+		return nil, fmt.Errorf("redis masterName must not be empty")
+	}
+	if len(cfg.SentinelAddrs) == 0 {
+		return nil, fmt.Errorf("redis sentinelAddrs must not be empty")
+	}
+	return redis.NewFailoverClient(&redis.FailoverOptions{
+		MasterName:    cfg.MasterName,
+		SentinelAddrs: cfg.SentinelAddrs,
+		Username:      cfg.Username,
+		Password:      cfg.Password,
+	}), nil
+}
+
 // UpdateStaticLayer updates the static layer a registry-facade adds
 func (reg *Registry) UpdateStaticLayer(ctx context.Context, cfg []config.StaticLayerCfg) error {
 	l, err := buildStaticLayer(ctx, cfg, reg.Resolver)

install/installer/pkg/components/registry-facade/configmap.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	wsmanager "github.com/gitpod-io/gitpod/installer/pkg/components/ws-manager"
+	"github.com/gitpod-io/gitpod/installer/pkg/config/v1/experimental"
 	regfac "github.com/gitpod-io/gitpod/registry-facade/api/config"
 
 	corev1 "k8s.io/api/core/v1"
@@ -25,6 +26,24 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 		}
 	}
 
+	var ipfsCache *regfac.IPFSCacheConfig
+	_ = ctx.WithExperimental(func(ucfg *experimental.Config) error {
+		if ucfg.Workspace == nil || !ucfg.Workspace.RegistryFacade.IPFSCache.Enabled {
+			return nil
+		}
+		cacheCfg := ucfg.Workspace.RegistryFacade.IPFSCache
+		ipfsCache = &regfac.IPFSCacheConfig{
+			Enabled:  true,
+			IPFSAddr: cacheCfg.IPFSAddr,
+			Redis: regfac.RedisConfig{
+				MasterName:    cacheCfg.Redis.MasterName,
+				SentinelAddrs: cacheCfg.Redis.SentinelAddrs,
+				Username:      cacheCfg.Redis.Username,
+			},
+		}
+		return nil
+	})
+
 	rfcfg := regfac.ServiceConfig{
 		Registry: regfac.Config{
 			Port: ContainerPort,
@@ -53,6 +72,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 					Type: "image",
 				},
 			},
+			IPFSCache: ipfsCache,
 		},
 		AuthCfg:            "/mnt/pull-secret.json",
 		PProfAddr:          ":6060",

install/installer/pkg/components/registry-facade/daemonset.go

@@ -11,6 +11,7 @@ import (
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry"
 	wsmanager "github.com/gitpod-io/gitpod/installer/pkg/components/ws-manager"
+	"github.com/gitpod-io/gitpod/installer/pkg/config/v1/experimental"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -31,8 +32,10 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
 		hashObj = append(hashObj, objs...)
 	}
 
-	var volumes []corev1.Volume
-	var volumeMounts []corev1.VolumeMount
+	var (
+		volumes      []corev1.Volume
+		volumeMounts []corev1.VolumeMount
+	)
 
 	if ctx.Config.Certificate.Name != "" {
 		name := "config-certificates"
@@ -72,6 +75,39 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
 		return nil, fmt.Errorf("%s: invalid container registry config", Component)
 	}
 
+	var envvars []corev1.EnvVar
+	_ = ctx.WithExperimental(func(ucfg *experimental.Config) error {
+		if ucfg.Workspace == nil || !ucfg.Workspace.RegistryFacade.IPFSCache.Enabled {
+			return nil
+		}
+
+		envvars = []corev1.EnvVar{
+			{
+				Name: "IPFS_HOST",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "status.hostIP",
+					},
+				},
+			},
+		}
+		if scr := ucfg.Workspace.RegistryFacade.IPFSCache.Redis.PasswordSecret; scr != "" {
+			envvars = append(envvars, corev1.EnvVar{
+				Name: "REDIS_PASSWORD",
+				ValueFrom: &corev1.EnvVarSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: scr,
+						},
+						Key: "password",
+					},
+				},
+			})
+		}
+
+		return nil
+	})
+
 	return []runtime.Object{&appsv1.DaemonSet{
 		TypeMeta: common.TypeMetaDaemonset,
 		ObjectMeta: metav1.ObjectMeta{
@@ -152,36 +188,46 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
 						Env: common.MergeEnv(
 							common.DefaultEnv(&ctx.Config),
 							common.TracingEnv(ctx),
-							[]corev1.EnvVar{{
-								Name:  "GRPC_GO_RETRY",
-								Value: "on",
-							}, {
-								Name: "NODENAME",
-								ValueFrom: &corev1.EnvVarSource{
-									FieldRef: &corev1.ObjectFieldSelector{
-										FieldPath: "spec.nodeName",
+							[]corev1.EnvVar{
+								{
+									Name:  "GRPC_GO_RETRY",
+									Value: "on",
+								},
+								{
+									Name: "NODENAME",
+									ValueFrom: &corev1.EnvVarSource{
+										FieldRef: &corev1.ObjectFieldSelector{
+											FieldPath: "spec.nodeName",
+										},
 									},
 								},
-							}},
+							},
+						),
+						VolumeMounts: append(
+							[]corev1.VolumeMount{
+								{
+									Name:      "cache",
+									MountPath: "/mnt/cache",
+								},
+								{
+									Name:      "config",
+									MountPath: "/mnt/config",
+									ReadOnly:  true,
+								},
+								{
+									Name:      "ws-manager-client-tls-certs",
+									MountPath: "/ws-manager-client-tls-certs",
+									ReadOnly:  true,
+								},
+								{
+									Name:      name,
+									MountPath: "/mnt/pull-secret.json",
+									SubPath:   ".dockerconfigjson",
+								},
+								*common.InternalCAVolumeMount(),
+							},
+							volumeMounts...,
 						),
-						VolumeMounts: append([]corev1.VolumeMount{{
-							Name:      "cache",
-							MountPath: "/mnt/cache",
-						}, {
-							Name:      "config",
-							MountPath: "/mnt/config",
-							ReadOnly:  true,
-						}, {
-							Name:      "ws-manager-client-tls-certs",
-							MountPath: "/ws-manager-client-tls-certs",
-							ReadOnly:  true,
-						}, {
-							Name:      name,
-							MountPath: "/mnt/pull-secret.json",
-							SubPath:   ".dockerconfigjson",
-						},
-							*common.InternalCAVolumeMount(),
-						}, volumeMounts...),
 						ReadinessProbe: &corev1.Probe{
 							ProbeHandler: corev1.ProbeHandler{
 								HTTPGet: &corev1.HTTPGetAction{

install/installer/pkg/config/v1/experimental/experimental.go

@@ -29,6 +29,19 @@ type WorkspaceConfig struct {
 		Limit            resource.Quantity `json:"limit"`
 		BurstLimit       resource.Quantity `json:"burstLimit"`
 	}
+
+	RegistryFacade struct {
+		IPFSCache struct {
+			Enabled  bool   `json:"enabled"`
+			IPFSAddr string `json:"ipfsAddr"`
+			Redis    struct {
+				MasterName     string   `json:"masterName"`
+				SentinelAddrs  []string `json:"sentinelAddrs"`
+				Username       string   `json:"username"`
+				PasswordSecret string   `json:"passwordSecret"`
+			} `json:"redis"`
+		} `json:"ipfsCache"`
+	} `json:"registryFacade"`
 }
 
 type WebAppConfig struct {

install/installer/pkg/config/v1/experimental/validation.go

@@ -4,7 +4,10 @@
 
 package experimental
 
-import "github.com/go-playground/validator/v10"
+import (
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
+	"github.com/go-playground/validator/v10"
+)
 
 var TracingSampleTypeList = map[TracingSampleType]struct{}{
 	TracingSampleTypeConst:         {},
@@ -19,3 +22,17 @@ var ValidationChecks = map[string]validator.Func{
 		return ok
 	},
 }
+
+func ClusterValidation(cfg *Config) cluster.ValidationChecks {
+	if cfg == nil {
+		return nil
+	}
+
+	var res cluster.ValidationChecks
+	if cfg.Workspace != nil {
+		if scr := cfg.Workspace.RegistryFacade.IPFSCache.Redis.PasswordSecret; scr != "" {
+			res = append(res, cluster.CheckSecret(scr, cluster.CheckSecretRequiredData("password")))
+		}
+	}
+	return res
+}

install/installer/pkg/config/v1/validation.go

@@ -210,5 +210,8 @@ func (v version) ClusterValidation(rcfg interface{}) cluster.ValidationChecks {
 			return errors, nil
 		})))
 	}
+
+	res = append(res, experimental.ClusterValidation(cfg.Experimental)...)
+
 	return res
 }