From f3435fbf444112a7bd5d9a002f54b50bbec6f91b Mon Sep 17 00:00:00 2001 From: Christian Weichel Date: Fri, 13 May 2022 12:58:21 +0000 Subject: [PATCH] [ws-daemon] Use baseserver to run services --- components/ws-daemon/cmd/run.go | 132 ++++-------------- components/ws-daemon/go.mod | 2 +- components/ws-daemon/pkg/config/config.go | 45 +----- .../pkg/components/ws-daemon/configmap.go | 20 +-- .../pkg/components/ws-daemon/constants.go | 4 +- .../pkg/components/ws-manager/deployment.go | 4 +- 6 files changed, 46 insertions(+), 161 deletions(-) diff --git a/components/ws-daemon/cmd/run.go b/components/ws-daemon/cmd/run.go index c4a6964013b778..b096d4da06a6ae 100644 --- a/components/ws-daemon/cmd/run.go +++ b/components/ws-daemon/cmd/run.go @@ -7,33 +7,26 @@ package cmd import ( "context" "fmt" - "net" - "net/http" - "os" - "os/signal" - "syscall" + "google.golang.org/grpc/credentials/insecure" "time" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/heptiolabs/healthcheck" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" - "golang.org/x/xerrors" "google.golang.org/grpc" "google.golang.org/grpc/credentials" - "google.golang.org/grpc/health" "google.golang.org/grpc/health/grpc_health_v1" + "github.com/gitpod-io/gitpod/common-go/baseserver" common_grpc "github.com/gitpod-io/gitpod/common-go/grpc" "github.com/gitpod-io/gitpod/common-go/log" - "github.com/gitpod-io/gitpod/common-go/pprof" "github.com/gitpod-io/gitpod/common-go/watch" "github.com/gitpod-io/gitpod/ws-daemon/pkg/config" "github.com/gitpod-io/gitpod/ws-daemon/pkg/daemon" ) +const grpcServerName = "wsdaemon" + // serveCmd represents the serve command var runCmd = &cobra.Command{ Use: "run", @@ -42,91 +35,31 @@ var runCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { cfg, err := config.Read(configFile) if err != nil { - log.WithError(err).Fatal("cannot read configuration. Maybe missing --config?") - } - reg := prometheus.NewRegistry() - dmn, err := daemon.NewDaemon(cfg.Daemon, prometheus.WrapRegistererWithPrefix("gitpod_ws_daemon_", reg)) - if err != nil { - log.WithError(err).Fatal("cannot create daemon") + log.WithError(err).Fatal("Cannot read configuration. Maybe missing --config?") } - common_grpc.SetupLogging() - - grpcMetrics := grpc_prometheus.NewServerMetrics() - grpcMetrics.EnableHandlingTimeHistogram() - reg.MustRegister(grpcMetrics) - - grpcOpts := common_grpc.ServerOptionsWithInterceptors( - []grpc.StreamServerInterceptor{grpcMetrics.StreamServerInterceptor()}, - []grpc.UnaryServerInterceptor{grpcMetrics.UnaryServerInterceptor()}, + health := healthcheck.NewHandler() + srv, err := baseserver.New(grpcServerName, + baseserver.WithGRPC(&cfg.Service), + baseserver.WithHealthHandler(health), ) - tlsOpt, err := cfg.Service.TLS.ServerOption() if err != nil { - log.WithError(err).Fatal("cannot use TLS config") + log.WithError(err).Fatal("Cannot set up server.") } - if tlsOpt != nil { - log.WithField("crt", cfg.Service.TLS.Certificate).WithField("key", cfg.Service.TLS.PrivateKey).Debug("securing gRPC server with TLS") - grpcOpts = append(grpcOpts, tlsOpt) - } else { - log.Warn("no TLS configured - gRPC server will be unsecured") - } - - healthServer := health.NewServer() - server := grpc.NewServer(grpcOpts...) - server.RegisterService(&grpc_health_v1.Health_ServiceDesc, healthServer) - - dmn.Register(server) - lis, err := net.Listen("tcp", cfg.Service.Addr) + dmn, err := daemon.NewDaemon(cfg.Daemon, prometheus.WrapRegistererWithPrefix("gitpod_ws_daemon_", srv.MetricsRegistry())) if err != nil { - log.WithError(err).Fatalf("cannot listen on %s", cfg.Service.Addr) + log.WithError(err).Fatal("Cannot create daemon.") } - go func() { - err := server.Serve(lis) - if err != nil { - log.WithError(err).Fatal("cannot start server") - } - }() - log.WithField("addr", cfg.Service.Addr).Info("started gRPC server") - - if cfg.Prometheus.Addr != "" { - reg.MustRegister( - collectors.NewGoCollector(), - collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), - ) - - handler := http.NewServeMux() - handler.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) - go func() { - err := http.ListenAndServe(cfg.Prometheus.Addr, handler) - if err != nil { - log.WithError(err).Error("Prometheus metrics server failed") - } - }() - log.WithField("addr", cfg.Prometheus.Addr).Info("started Prometheus metrics server") - } - - if cfg.PProf.Addr != "" { - go pprof.Serve(cfg.PProf.Addr) - } + health.AddReadinessCheck("grpc-server", grpcProbe(cfg.Service)) + health.AddReadinessCheck("ws-daemon", dmn.ReadinessProbe()) - if cfg.ReadinessProbeAddr != "" { - // Ensure we can access the GRPC server is healthy, the etc hosts file was updated and containerd is available. - health := healthcheck.NewHandler() - health.AddReadinessCheck("grpc-server", grpcProbe(cfg.Service)) - health.AddReadinessCheck("ws-daemon", dmn.ReadinessProbe()) - - go func() { - if err := http.ListenAndServe(cfg.ReadinessProbeAddr, health); err != nil && err != http.ErrServerClosed { - log.WithError(err).Panic("error starting HTTP server") - } - }() - } + dmn.Register(srv.GRPC()) err = dmn.Start() if err != nil { - log.WithError(err).Fatal("cannot start daemon") + log.WithError(err).Fatal("Cannot start daemon.") } ctx, cancel := context.WithCancel(context.Background()) @@ -138,30 +71,23 @@ var runCmd = &cobra.Command{ cfg, err := config.Read(configFile) if err != nil { - log.WithError(err).Warn("cannot reload configuration") + log.WithError(err).Warn("Cannot reload configuration.") return } err = dmn.ReloadConfig(ctx, &cfg.Daemon) if err != nil { - log.WithError(err).Warn("cannot reload configuration") + log.WithError(err).Warn("Cannot reload configuration.") } }) if err != nil { - log.WithError(err).Fatal("cannot start watch of configuration file") + log.WithError(err).Fatal("Cannot start watch of configuration file.") } - // run until we're told to stop - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - log.Info("🧫 ws-daemon is up and running. Stop with SIGINT or CTRL+C") - <-sigChan - server.Stop() - err = dmn.Stop() + err = srv.ListenAndServe() if err != nil { - log.WithError(err).Error("cannot shut down gracefully") + log.WithError(err).Fatal("Failed to listen and serve.") } - log.Info("Received SIGINT - shutting down") }, } @@ -169,26 +95,26 @@ func init() { rootCmd.AddCommand(runCmd) } -func grpcProbe(tlsConfig config.AddrTLS) func() error { +func grpcProbe(cfg baseserver.ServerConfiguration) func() error { return func() error { - secopt := grpc.WithInsecure() - if tlsConfig.TLS != nil && tlsConfig.TLS.Certificate != "" { + creds := insecure.NewCredentials() + if cfg.TLS != nil && cfg.TLS.CertPath != "" { tlsConfig, err := common_grpc.ClientAuthTLSConfig( - tlsConfig.TLS.Authority, tlsConfig.TLS.Certificate, tlsConfig.TLS.PrivateKey, + cfg.TLS.CAPath, cfg.TLS.CertPath, cfg.TLS.KeyPath, common_grpc.WithSetRootCAs(true), - common_grpc.WithServerName("wsdaemon"), + common_grpc.WithServerName(grpcServerName), ) if err != nil { - return xerrors.Errorf("cannot load ws-daemon certificate: %w", err) + return fmt.Errorf("cannot load ws-daemon certificate: %w", err) } - secopt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)) + creds = credentials.NewTLS(tlsConfig) } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - conn, err := grpc.DialContext(ctx, tlsConfig.Addr, secopt) + conn, err := grpc.DialContext(ctx, cfg.Address, grpc.WithTransportCredentials(creds)) if err != nil { return err } diff --git a/components/ws-daemon/go.mod b/components/ws-daemon/go.mod index 0ccd7b6f360ae6..85b941c20c4ebc 100644 --- a/components/ws-daemon/go.mod +++ b/components/ws-daemon/go.mod @@ -14,7 +14,7 @@ require ( github.com/gitpod-io/gitpod/ws-daemon/api v0.0.0-00010101000000-000000000000 github.com/google/go-cmp v0.5.8 github.com/google/uuid v1.3.0 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.2 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 diff --git a/components/ws-daemon/pkg/config/config.go b/components/ws-daemon/pkg/config/config.go index 86dcd03476256f..fa2d8d799fb151 100644 --- a/components/ws-daemon/pkg/config/config.go +++ b/components/ws-daemon/pkg/config/config.go @@ -6,15 +6,12 @@ package config import ( "bytes" - "crypto/tls" "encoding/json" "os" "golang.org/x/xerrors" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - common_grpc "github.com/gitpod-io/gitpod/common-go/grpc" + "github.com/gitpod-io/gitpod/common-go/baseserver" "github.com/gitpod-io/gitpod/ws-daemon/pkg/daemon" ) @@ -36,42 +33,6 @@ func Read(fn string) (*Config, error) { } type Config struct { - Daemon daemon.Config `json:"daemon"` - Service AddrTLS `json:"service"` - Prometheus Addr `json:"prometheus"` - PProf Addr `json:"pprof"` - ReadinessProbeAddr string `json:"readinessProbeAddr"` -} - -type Addr struct { - Addr string `json:"address"` -} - -type AddrTLS struct { - Addr string `json:"address"` - TLS *TLS `json:"tls,omitempty"` -} -type TLS struct { - Authority string `json:"ca"` - Certificate string `json:"crt"` - PrivateKey string `json:"key"` -} - -// ServerOption produces the GRPC option that configures a server to use this TLS configuration -func (c *TLS) ServerOption() (grpc.ServerOption, error) { - if c.Authority == "" || c.Certificate == "" || c.PrivateKey == "" { - return nil, nil - } - - tlsConfig, err := common_grpc.ClientAuthTLSConfig( - c.Authority, c.Certificate, c.PrivateKey, - common_grpc.WithClientAuth(tls.RequireAndVerifyClientCert), - common_grpc.WithSetClientCAs(true), - common_grpc.WithServerName("ws-manager"), - ) - if err != nil { - return nil, xerrors.Errorf("cannot load certs: %w", err) - } - - return grpc.Creds(credentials.NewTLS(tlsConfig)), nil + Daemon daemon.Config `json:"daemon"` + Service baseserver.ServerConfiguration `json:"service"` } diff --git a/install/installer/pkg/components/ws-daemon/configmap.go b/install/installer/pkg/components/ws-daemon/configmap.go index e45f6ddad19c7c..cfcbe5106fe636 100644 --- a/install/installer/pkg/components/ws-daemon/configmap.go +++ b/install/installer/pkg/components/ws-daemon/configmap.go @@ -8,6 +8,7 @@ import ( "fmt" "time" + "github.com/gitpod-io/gitpod/common-go/baseserver" "github.com/gitpod-io/gitpod/common-go/util" "github.com/gitpod-io/gitpod/installer/pkg/common" config "github.com/gitpod-io/gitpod/installer/pkg/config/v1" @@ -127,21 +128,14 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { }}, }, }, - Service: wsdconfig.AddrTLS{ - Addr: fmt.Sprintf(":%d", ServicePort), - TLS: &wsdconfig.TLS{ - Authority: "/certs/ca.crt", - Certificate: "/certs/tls.crt", - PrivateKey: "/certs/tls.key", + Service: baseserver.ServerConfiguration{ + Address: fmt.Sprintf(":%d", ServicePort), + TLS: &baseserver.TLSConfiguration{ + CAPath: "/certs/ca.crt", + CertPath: "/certs/tls.crt", + KeyPath: "/certs/tls.key", }, }, - Prometheus: wsdconfig.Addr{ - Addr: "localhost:9500", - }, - PProf: wsdconfig.Addr{ - Addr: "localhost:6060", - }, - ReadinessProbeAddr: fmt.Sprintf(":%v", ReadinessPort), } fc, err := common.ToJSONString(wsdcfg) if err != nil { diff --git a/install/installer/pkg/components/ws-daemon/constants.go b/install/installer/pkg/components/ws-daemon/constants.go index 937dafc7518831..87048f2d537579 100644 --- a/install/installer/pkg/components/ws-daemon/constants.go +++ b/install/installer/pkg/components/ws-daemon/constants.go @@ -4,6 +4,8 @@ package wsdaemon +import "github.com/gitpod-io/gitpod/common-go/baseserver" + const ( Component = "ws-daemon" ServicePort = 8080 @@ -12,5 +14,5 @@ const ( HostBackupPath = "/var/gitpod/tmp/backup" TLSSecretName = "ws-daemon-tls" VolumeTLSCerts = "ws-daemon-tls-certs" - ReadinessPort = 8086 + ReadinessPort = baseserver.BuiltinHealthPort ) diff --git a/install/installer/pkg/components/ws-manager/deployment.go b/install/installer/pkg/components/ws-manager/deployment.go index a56757d9295528..a97f0767684959 100644 --- a/install/installer/pkg/components/ws-manager/deployment.go +++ b/install/installer/pkg/components/ws-manager/deployment.go @@ -5,6 +5,8 @@ package wsmanager import ( + "fmt" + "github.com/gitpod-io/gitpod/common-go/baseserver" "github.com/gitpod-io/gitpod/installer/pkg/cluster" "github.com/gitpod-io/gitpod/installer/pkg/common" wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon" @@ -77,7 +79,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { }, }, }, - *common.KubeRBACProxyContainer(ctx), + *common.KubeRBACProxyContainerWithConfig(ctx, 9500, fmt.Sprintf("http://127.0.0.1:%d/", baseserver.BuiltinMetricsPort)), }, Volumes: []corev1.Volume{ {