[ws-daemon] Use baseserver to run its services

components/ws-daemon/cmd/run.go

@@ -7,33 +7,26 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"os/signal"
-	"syscall"
+	"google.golang.org/grpc/credentials/insecure"
 	"time"
 
-	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/heptiolabs/healthcheck"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/collectors"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
-	"golang.org/x/xerrors"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/health"
 	"google.golang.org/grpc/health/grpc_health_v1"
 
+	"github.com/gitpod-io/gitpod/common-go/baseserver"
 	common_grpc "github.com/gitpod-io/gitpod/common-go/grpc"
 	"github.com/gitpod-io/gitpod/common-go/log"
-	"github.com/gitpod-io/gitpod/common-go/pprof"
 	"github.com/gitpod-io/gitpod/common-go/watch"
 	"github.com/gitpod-io/gitpod/ws-daemon/pkg/config"
 	"github.com/gitpod-io/gitpod/ws-daemon/pkg/daemon"
 )
 
+const grpcServerName = "wsdaemon"
+
 // serveCmd represents the serve command
 var runCmd = &cobra.Command{
 	Use:   "run",
@@ -42,91 +35,31 @@ var runCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		cfg, err := config.Read(configFile)
 		if err != nil {
-			log.WithError(err).Fatal("cannot read configuration. Maybe missing --config?")
-		}
-		reg := prometheus.NewRegistry()
-		dmn, err := daemon.NewDaemon(cfg.Daemon, prometheus.WrapRegistererWithPrefix("gitpod_ws_daemon_", reg))
-		if err != nil {
-			log.WithError(err).Fatal("cannot create daemon")
+			log.WithError(err).Fatal("Cannot read configuration. Maybe missing --config?")
 		}
 
-		common_grpc.SetupLogging()
-
-		grpcMetrics := grpc_prometheus.NewServerMetrics()
-		grpcMetrics.EnableHandlingTimeHistogram()
-		reg.MustRegister(grpcMetrics)
-
-		grpcOpts := common_grpc.ServerOptionsWithInterceptors(
-			[]grpc.StreamServerInterceptor{grpcMetrics.StreamServerInterceptor()},
-			[]grpc.UnaryServerInterceptor{grpcMetrics.UnaryServerInterceptor()},
+		health := healthcheck.NewHandler()
+		srv, err := baseserver.New(grpcServerName,
+			baseserver.WithGRPC(&cfg.Service),
+			baseserver.WithHealthHandler(health),
 		)
-		tlsOpt, err := cfg.Service.TLS.ServerOption()
 		if err != nil {
-			log.WithError(err).Fatal("cannot use TLS config")
+			log.WithError(err).Fatal("Cannot set up server.")
 		}
-		if tlsOpt != nil {
-			log.WithField("crt", cfg.Service.TLS.Certificate).WithField("key", cfg.Service.TLS.PrivateKey).Debug("securing gRPC server with TLS")
-			grpcOpts = append(grpcOpts, tlsOpt)
-		} else {
-			log.Warn("no TLS configured - gRPC server will be unsecured")
-		}
-
-		healthServer := health.NewServer()
 
-		server := grpc.NewServer(grpcOpts...)
-		server.RegisterService(&grpc_health_v1.Health_ServiceDesc, healthServer)
-
-		dmn.Register(server)
-		lis, err := net.Listen("tcp", cfg.Service.Addr)
+		dmn, err := daemon.NewDaemon(cfg.Daemon, prometheus.WrapRegistererWithPrefix("gitpod_ws_daemon_", srv.MetricsRegistry()))
 		if err != nil {
-			log.WithError(err).Fatalf("cannot listen on %s", cfg.Service.Addr)
+			log.WithError(err).Fatal("Cannot create daemon.")
 		}
-		go func() {
-			err := server.Serve(lis)
-			if err != nil {
-				log.WithError(err).Fatal("cannot start server")
-			}
-		}()
-		log.WithField("addr", cfg.Service.Addr).Info("started gRPC server")
-
-		if cfg.Prometheus.Addr != "" {
-			reg.MustRegister(
-				collectors.NewGoCollector(),
-				collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
-			)
-
-			handler := http.NewServeMux()
-			handler.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
 
-			go func() {
-				err := http.ListenAndServe(cfg.Prometheus.Addr, handler)
-				if err != nil {
-					log.WithError(err).Error("Prometheus metrics server failed")
-				}
-			}()
-			log.WithField("addr", cfg.Prometheus.Addr).Info("started Prometheus metrics server")
-		}
-
-		if cfg.PProf.Addr != "" {
-			go pprof.Serve(cfg.PProf.Addr)
-		}
+		health.AddReadinessCheck("grpc-server", grpcProbe(cfg.Service))
+		health.AddReadinessCheck("ws-daemon", dmn.ReadinessProbe())
 
-		if cfg.ReadinessProbeAddr != "" {
-			// Ensure we can access the GRPC server is healthy, the etc hosts file was updated and containerd is available.
-			health := healthcheck.NewHandler()
-			health.AddReadinessCheck("grpc-server", grpcProbe(cfg.Service))
-			health.AddReadinessCheck("ws-daemon", dmn.ReadinessProbe())
-
-			go func() {
-				if err := http.ListenAndServe(cfg.ReadinessProbeAddr, health); err != nil && err != http.ErrServerClosed {
-					log.WithError(err).Panic("error starting HTTP server")
-				}
-			}()
-		}
+		dmn.Register(srv.GRPC())
 
 		err = dmn.Start()
 		if err != nil {
-			log.WithError(err).Fatal("cannot start daemon")
+			log.WithError(err).Fatal("Cannot start daemon.")
 		}
 
 		ctx, cancel := context.WithCancel(context.Background())
@@ -138,57 +71,50 @@ var runCmd = &cobra.Command{
 
 			cfg, err := config.Read(configFile)
 			if err != nil {
-				log.WithError(err).Warn("cannot reload configuration")
+				log.WithError(err).Warn("Cannot reload configuration.")
 				return
 			}
 
 			err = dmn.ReloadConfig(ctx, &cfg.Daemon)
 			if err != nil {
-				log.WithError(err).Warn("cannot reload configuration")
+				log.WithError(err).Warn("Cannot reload configuration.")
 			}
 		})
 		if err != nil {
-			log.WithError(err).Fatal("cannot start watch of configuration file")
+			log.WithError(err).Fatal("Cannot start watch of configuration file.")
 		}
 
-		// run until we're told to stop
-		sigChan := make(chan os.Signal, 1)
-		signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
-		log.Info("🧫 ws-daemon is up and running. Stop with SIGINT or CTRL+C")
-		<-sigChan
-		server.Stop()
-		err = dmn.Stop()
+		err = srv.ListenAndServe()
 		if err != nil {
-			log.WithError(err).Error("cannot shut down gracefully")
+			log.WithError(err).Fatal("Failed to listen and serve.")
 		}
-		log.Info("Received SIGINT - shutting down")
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(runCmd)
 }
 
-func grpcProbe(tlsConfig config.AddrTLS) func() error {
+func grpcProbe(cfg baseserver.ServerConfiguration) func() error {
 	return func() error {
-		secopt := grpc.WithInsecure()
-		if tlsConfig.TLS != nil && tlsConfig.TLS.Certificate != "" {
+		creds := insecure.NewCredentials()
+		if cfg.TLS != nil && cfg.TLS.CertPath != "" {
 			tlsConfig, err := common_grpc.ClientAuthTLSConfig(
-				tlsConfig.TLS.Authority, tlsConfig.TLS.Certificate, tlsConfig.TLS.PrivateKey,
+				cfg.TLS.CAPath, cfg.TLS.CertPath, cfg.TLS.KeyPath,
 				common_grpc.WithSetRootCAs(true),
-				common_grpc.WithServerName("wsdaemon"),
+				common_grpc.WithServerName(grpcServerName),
 			)
 			if err != nil {
-				return xerrors.Errorf("cannot load ws-daemon certificate: %w", err)
+				return fmt.Errorf("cannot load ws-daemon certificate: %w", err)
 			}
 
-			secopt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
+			creds = credentials.NewTLS(tlsConfig)
 		}
 
 		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancel()
 
-		conn, err := grpc.DialContext(ctx, tlsConfig.Addr, secopt)
+		conn, err := grpc.DialContext(ctx, cfg.Address, grpc.WithTransportCredentials(creds))
 		if err != nil {
 			return err
 		}

components/ws-daemon/go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/gitpod-io/gitpod/ws-daemon/api v0.0.0-00010101000000-000000000000
 	github.com/google/go-cmp v0.5.8
 	github.com/google/uuid v1.3.0
-	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.2
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417

components/ws-daemon/pkg/config/config.go

@@ -6,15 +6,12 @@ package config
 
 import (
 	"bytes"
-	"crypto/tls"
 	"encoding/json"
 	"os"
 
 	"golang.org/x/xerrors"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials"
 
-	common_grpc "github.com/gitpod-io/gitpod/common-go/grpc"
+	"github.com/gitpod-io/gitpod/common-go/baseserver"
 	"github.com/gitpod-io/gitpod/ws-daemon/pkg/daemon"
 )
 
@@ -36,42 +33,6 @@ func Read(fn string) (*Config, error) {
 }
 
 type Config struct {
-	Daemon             daemon.Config `json:"daemon"`
-	Service            AddrTLS       `json:"service"`
-	Prometheus         Addr          `json:"prometheus"`
-	PProf              Addr          `json:"pprof"`
-	ReadinessProbeAddr string        `json:"readinessProbeAddr"`
-}
-
-type Addr struct {
-	Addr string `json:"address"`
-}
-
-type AddrTLS struct {
-	Addr string `json:"address"`
-	TLS  *TLS   `json:"tls,omitempty"`
-}
-type TLS struct {
-	Authority   string `json:"ca"`
-	Certificate string `json:"crt"`
-	PrivateKey  string `json:"key"`
-}
-
-// ServerOption produces the GRPC option that configures a server to use this TLS configuration
-func (c *TLS) ServerOption() (grpc.ServerOption, error) {
-	if c.Authority == "" || c.Certificate == "" || c.PrivateKey == "" {
-		return nil, nil
-	}
-
-	tlsConfig, err := common_grpc.ClientAuthTLSConfig(
-		c.Authority, c.Certificate, c.PrivateKey,
-		common_grpc.WithClientAuth(tls.RequireAndVerifyClientCert),
-		common_grpc.WithSetClientCAs(true),
-		common_grpc.WithServerName("ws-manager"),
-	)
-	if err != nil {
-		return nil, xerrors.Errorf("cannot load certs: %w", err)
-	}
-
-	return grpc.Creds(credentials.NewTLS(tlsConfig)), nil
+	Daemon  daemon.Config                  `json:"daemon"`
+	Service baseserver.ServerConfiguration `json:"service"`
 }

install/installer/pkg/components/ws-daemon/configmap.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/gitpod-io/gitpod/common-go/baseserver"
 	"github.com/gitpod-io/gitpod/common-go/util"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	config "github.com/gitpod-io/gitpod/installer/pkg/config/v1"
@@ -127,21 +128,14 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 				}},
 			},
 		},
-		Service: wsdconfig.AddrTLS{
-			Addr: fmt.Sprintf(":%d", ServicePort),
-			TLS: &wsdconfig.TLS{
-				Authority:   "/certs/ca.crt",
-				Certificate: "/certs/tls.crt",
-				PrivateKey:  "/certs/tls.key",
+		Service: baseserver.ServerConfiguration{
+			Address: fmt.Sprintf(":%d", ServicePort),
+			TLS: &baseserver.TLSConfiguration{
+				CAPath:   "/certs/ca.crt",
+				CertPath: "/certs/tls.crt",
+				KeyPath:  "/certs/tls.key",
 			},
 		},
-		Prometheus: wsdconfig.Addr{
-			Addr: "localhost:9500",
-		},
-		PProf: wsdconfig.Addr{
-			Addr: "localhost:6060",
-		},
-		ReadinessProbeAddr: fmt.Sprintf(":%v", ReadinessPort),
 	}
 	fc, err := common.ToJSONString(wsdcfg)
 	if err != nil {

install/installer/pkg/components/ws-daemon/constants.go

@@ -4,6 +4,8 @@
 
 package wsdaemon
 
+import "github.com/gitpod-io/gitpod/common-go/baseserver"
+
 const (
 	Component            = "ws-daemon"
 	ServicePort          = 8080
@@ -12,5 +14,5 @@ const (
 	HostBackupPath       = "/var/gitpod/tmp/backup"
 	TLSSecretName        = "ws-daemon-tls"
 	VolumeTLSCerts       = "ws-daemon-tls-certs"
-	ReadinessPort        = 8086
+	ReadinessPort        = baseserver.BuiltinHealthPort
 )

install/installer/pkg/components/ws-manager/deployment.go

@@ -5,6 +5,8 @@
 package wsmanager
 
 import (
+	"fmt"
+	"github.com/gitpod-io/gitpod/common-go/baseserver"
 	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
@@ -77,7 +79,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 				},
 			},
 		},
-			*common.KubeRBACProxyContainer(ctx),
+			*common.KubeRBACProxyContainerWithConfig(ctx, 9500, fmt.Sprintf("http://127.0.0.1:%d/", baseserver.BuiltinMetricsPort)),
 		},
 		Volumes: []corev1.Volume{
 			{