[baseserver] Dedicated debug server

Author: easyCZ

components/common-go/baseserver/options.go

@@ -18,6 +18,8 @@ type config struct {
 
 	// hostname is the hostname on which our servers will listen.
 	hostname string
+	// debugPort is the port we listen on for metrics, pprof, readiness and livenss checks
+	debugPort int
 	// grpcPort is the port we listen on for gRPC traffic
 	grpcPort int
 	// httpPort is the port we listen on for HTTP traffic
@@ -36,8 +38,9 @@ func defaultConfig() *config {
 	return &config{
 		logger:        log.New(),
 		hostname:      "localhost",
-		httpPort:      9000,
-		grpcPort:      9001,
+		httpPort:      -1, // disabled by default
+		grpcPort:      -1, // disabled by default
+		debugPort:     9500,
 		closeTimeout:  5 * time.Second,
 		healthHandler: healthcheck.NewHandler(),
 	}
@@ -52,24 +55,29 @@ func WithHostname(hostname string) Option {
 	}
 }
 
+// WithHTTPPort sets the port to use for an HTTP server. Setting WithHTTPPort also enables an HTTP server on the baseserver.
 func WithHTTPPort(port int) Option {
 	return func(cfg *config) error {
-		if port < 0 {
-			return fmt.Errorf("http must not be negative, got: %d", port)
-		}
-
 		cfg.httpPort = port
 		return nil
 	}
 }
 
+// WithGRPCPort sets the port to use for an HTTP server. Setting WithGRPCPort also enables a gRPC server on the baseserver.
 func WithGRPCPort(port int) Option {
+	return func(cfg *config) error {
+		cfg.grpcPort = port
+		return nil
+	}
+}
+
+func WithDebugPort(port int) Option {
 	return func(cfg *config) error {
 		if port < 0 {
 			return fmt.Errorf("grpc port must not be negative, got: %d", port)
 		}
 
-		cfg.grpcPort = port
+		cfg.debugPort = port
 		return nil
 	}
 }

components/common-go/baseserver/options_test.go

@@ -17,13 +17,15 @@ func TestOptions(t *testing.T) {
 	logger := log.New()
 	httpPort := 8080
 	grpcPort := 8081
+	debugPort := 8082
 	timeout := 10 * time.Second
 	hostname := "another_hostname"
 	registry := prometheus.NewRegistry()
 	health := healthcheck.NewHandler()
 
 	var opts = []Option{
 		WithHostname(hostname),
+		WithDebugPort(debugPort),
 		WithHTTPPort(httpPort),
 		WithGRPCPort(grpcPort),
 		WithLogger(logger),
@@ -39,32 +41,41 @@ func TestOptions(t *testing.T) {
 		hostname:        hostname,
 		grpcPort:        grpcPort,
 		httpPort:        httpPort,
+		debugPort:       debugPort,
 		closeTimeout:    timeout,
 		metricsRegistry: registry,
 		healthHandler:   health,
 	}, cfg)
 }
 
-func TestWithTTPPort(t *testing.T) {
-	t.Run("negative", func(t *testing.T) {
-		_, err := evaluateOptions(defaultConfig(), WithHTTPPort(-1))
-		require.Error(t, err)
-	})
+func TestWithHTTPPort(t *testing.T) {
+	cfg, err := evaluateOptions(defaultConfig(), WithHTTPPort(-1))
+	require.NoError(t, err)
+	require.Equal(t, -1, cfg.httpPort)
 
-	t.Run("zero", func(t *testing.T) {
-		_, err := evaluateOptions(defaultConfig(), WithHTTPPort(0))
-		require.NoError(t, err)
-	})
+	cfg, err = evaluateOptions(defaultConfig(), WithHTTPPort(0))
+	require.NoError(t, err)
+	require.Equal(t, 0, cfg.httpPort)
 }
 
 func TestWithGRPCPort(t *testing.T) {
+	cfg, err := evaluateOptions(defaultConfig(), WithGRPCPort(-1))
+	require.NoError(t, err)
+	require.Equal(t, -1, cfg.grpcPort)
+
+	cfg, err = evaluateOptions(defaultConfig(), WithGRPCPort(0))
+	require.NoError(t, err)
+	require.Equal(t, 0, cfg.grpcPort)
+}
+
+func TestWithDebugPort(t *testing.T) {
 	t.Run("negative", func(t *testing.T) {
-		_, err := evaluateOptions(defaultConfig(), WithGRPCPort(-1))
+		_, err := evaluateOptions(defaultConfig(), WithDebugPort(-1))
 		require.Error(t, err)
 	})
 
 	t.Run("zero", func(t *testing.T) {
-		_, err := evaluateOptions(defaultConfig(), WithGRPCPort(0))
+		_, err := evaluateOptions(defaultConfig(), WithDebugPort(0))
 		require.NoError(t, err)
 	})
 }

components/common-go/baseserver/server.go

@@ -31,28 +31,57 @@ func New(name string, opts ...Option) (*Server, error) {
 		cfg:  cfg,
 	}
 
-	if initErr := server.initializeHTTP(); initErr != nil {
-		return nil, fmt.Errorf("failed to initialize http server: %w", initErr)
+	if initErr := server.initializeDebug(); initErr != nil {
+		return nil, fmt.Errorf("failed to initialize debug server: %w", initErr)
 	}
-	if initErr := server.initializeGRPC(); initErr != nil {
-		return nil, fmt.Errorf("failed to initialize grpc server: %w", initErr)
+
+	if server.isHTTPEnabled() {
+		if initErr := server.initializeHTTP(); initErr != nil {
+			return nil, fmt.Errorf("failed to initialize http server: %w", initErr)
+		}
+	}
+
+	if server.isGRPCEnabled() {
+		if initErr := server.initializeGRPC(); initErr != nil {
+			return nil, fmt.Errorf("failed to initialize grpc server: %w", initErr)
+		}
 	}
 
 	return server, nil
 }
 
+// Server is a packaged server with batteries included. It is designed to be standard across components where it makes sense.
+// Server implements graceful shutdown making it suitable for usage in integration tests. See server_test.go.
+//
+// Server is composed of the following:
+// 	* Debug server which serves observability and debug endpoints
+//		- /metrics for Prometheus metrics
+//		- /pprof for Golang profiler
+//		- /ready for kubernetes readiness check
+//		- /live for kubernetes liveness check
+//	* (optional) gRPC server with standard interceptors and configuration
+//		- Started when baseserver is configured WithGRPCPort (port is non-negative)
+//		- Use Server.GRPC() to get access to the underlying grpc.Server and register services
+//	* (optional) HTTP server
+//		- Currently does not come with any standard HTTP middlewares
+//		- Started when baseserver is configured WithHTTPPort (port is non-negative)
+// 		- Use Server.HTTPMux() to get access to the root handler and register your endpoints
 type Server struct {
 	// Name is the name of this server, used for logging context
 	Name string
 
 	cfg *config
 
-	// http is an http Server
+	// debug is an HTTP server for debug endpoints - metrics, pprof, readiness & liveness.
+	debug         *http.Server
+	debugListener net.Listener
+
+	// http is an http Server, only used when port is specified in cfg
 	http         *http.Server
 	httpMux      *http.ServeMux
 	httpListener net.Listener
 
-	// grpc is a grpc Server
+	// grpc is a grpc Server, only used when port is specified in cfg
 	grpc         *grpc.Server
 	grpcListener net.Listener
 
@@ -62,25 +91,34 @@ type Server struct {
 
 func (s *Server) ListenAndServe() error {
 	var err error
-	s.grpcListener, err = net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.grpcPort))
+
+	s.debugListener, err = net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.debugPort))
 	if err != nil {
-		return fmt.Errorf("failed to acquire port %d", s.cfg.grpcPort)
+		return fmt.Errorf("failed to acquire port %d", s.cfg.debugPort)
 	}
 
-	s.httpListener, err = net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.httpPort))
-	if err != nil {
-		return fmt.Errorf("failed to acquire port %d", s.cfg.httpPort)
+	if s.isGRPCEnabled() {
+		s.grpcListener, err = net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.grpcPort))
+		if err != nil {
+			return fmt.Errorf("failed to acquire port %d", s.cfg.grpcPort)
+		}
+	}
+
+	if s.isHTTPEnabled() {
+		s.httpListener, err = net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.httpPort))
+		if err != nil {
+			return fmt.Errorf("failed to acquire port %d", s.cfg.httpPort)
+		}
 	}
 
 	errors := make(chan error)
 	defer close(errors)
 	s.listening = make(chan struct{})
 
+	// Always start the debug server, we should always have metrics and other debug information.
 	go func() {
-		s.Logger().
-			WithField("protocol", "grpc").
-			Infof("Serving gRPC on %s", s.grpcListener.Addr().String())
-		if serveErr := s.grpc.Serve(s.grpcListener); serveErr != nil {
+		s.Logger().WithField("protocol", "http").Infof("Serving debug server on %s", s.debugListener.Addr().String())
+		if serveErr := s.debug.Serve(s.debugListener); serveErr != nil {
 			if s.isClosing() {
 				return
 			}
@@ -89,18 +127,31 @@ func (s *Server) ListenAndServe() error {
 		}
 	}()
 
-	go func() {
-		s.Logger().
-			WithField("protocol", "http").
-			Infof("Serving http on %s", s.httpListener.Addr().String())
-		if serveErr := s.http.Serve(s.httpListener); serveErr != nil {
-			if s.isClosing() {
-				return
+	if s.isGRPCEnabled() {
+		go func() {
+			s.Logger().WithField("protocol", "grpc").Infof("Serving gRPC on %s", s.grpcListener.Addr().String())
+			if serveErr := s.grpc.Serve(s.grpcListener); serveErr != nil {
+				if s.isClosing() {
+					return
+				}
+
+				errors <- serveErr
 			}
+		}()
+	}
 
-			errors <- serveErr
-		}
-	}()
+	if s.isHTTPEnabled() {
+		go func() {
+			s.Logger().WithField("protocol", "http").Infof("Serving http on %s", s.httpListener.Addr().String())
+			if serveErr := s.http.Serve(s.httpListener); serveErr != nil {
+				if s.isClosing() {
+					return
+				}
+
+				errors <- serveErr
+			}
+		}()
+	}
 
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
@@ -157,6 +208,15 @@ func (s *Server) GRPCAddress() string {
 	return fmt.Sprintf("%s:%d", s.cfg.hostname, addr.Port)
 }
 
+func (s *Server) DebugAddress() string {
+	if s.debugListener == nil {
+		return ""
+	}
+	protocol := "http"
+	addr := s.debugListener.Addr().(*net.TCPAddr)
+	return fmt.Sprintf("%s://%s:%d", protocol, s.cfg.hostname, addr.Port)
+}
+
 func (s *Server) HTTPMux() *http.ServeMux {
 	return s.httpMux
 }
@@ -178,17 +238,29 @@ func (s *Server) close(ctx context.Context) error {
 	s.Logger().Info("Received graceful shutdown request.")
 	close(s.listening)
 
-	s.grpc.GracefulStop()
-	// s.grpc.GracefulStop() also closes the underlying net.Listener, we just release the reference.
-	s.grpcListener = nil
-	s.Logger().Info("GRPC server terminated.")
+	if s.isGRPCEnabled() {
+		s.grpc.GracefulStop()
+		// s.grpc.GracefulStop() also closes the underlying net.Listener, we just release the reference.
+		s.grpcListener = nil
+		s.Logger().Info("GRPC server terminated.")
+	}
 
-	if err := s.http.Shutdown(ctx); err != nil {
-		return fmt.Errorf("failed to close http server: %w", err)
+	if s.isHTTPEnabled() {
+		if err := s.http.Shutdown(ctx); err != nil {
+			return fmt.Errorf("failed to close http server: %w", err)
+		}
+		// s.http.Shutdown() also closes the underlying net.Listener, we just release the reference.
+		s.httpListener = nil
+		s.Logger().Info("HTTP server terminated.")
+	}
+
+	// Always terminate debug server last, we want to keep it running for as long as possible
+	if err := s.debug.Shutdown(ctx); err != nil {
+		return fmt.Errorf("failed to close debug server: %w", err)
 	}
 	// s.http.Shutdown() also closes the underlying net.Listener, we just release the reference.
-	s.httpListener = nil
-	s.Logger().Info("HTTP server terminated.")
+	s.debugListener = nil
+	s.Logger().Info("Debug server terminated.")
 
 	return nil
 }
@@ -204,7 +276,7 @@ func (s *Server) isClosing() bool {
 }
 
 func (s *Server) initializeHTTP() error {
-	s.httpMux = s.newHTTPMux()
+	s.httpMux = http.NewServeMux()
 	s.http = &http.Server{
 		Addr:    fmt.Sprintf(":%d", s.cfg.httpPort),
 		Handler: s.httpMux,
@@ -213,13 +285,16 @@ func (s *Server) initializeHTTP() error {
 	return nil
 }
 
-func (s *Server) newHTTPMux() *http.ServeMux {
+func (s *Server) initializeDebug() error {
+	logger := s.Logger().WithField("protocol", "debug")
+
 	mux := http.NewServeMux()
+
 	mux.HandleFunc("/ready", s.cfg.healthHandler.ReadyEndpoint)
-	s.Logger().WithField("protocol", "http").Debug("Serving readiness handler on /ready")
+	logger.Debug("Serving readiness handler on /ready")
 
 	mux.HandleFunc("/live", s.cfg.healthHandler.LiveEndpoint)
-	s.Logger().WithField("protocol", "http").Debug("Serving liveliness handler on /live")
+	logger.Debug("Serving liveliness handler on /live")
 
 	// Metrics endpoint
 	metricsHandler := promhttp.Handler()
@@ -229,12 +304,17 @@ func (s *Server) newHTTPMux() *http.ServeMux {
 		)
 	}
 	mux.Handle("/metrics", metricsHandler)
-	s.Logger().WithField("protocol", "http").Debug("Serving metrics on /metrics")
+	logger.Debug("Serving metrics on /metrics")
 
 	mux.Handle(pprof.Path, pprof.Handler())
-	s.Logger().WithField("protocol", "http").Debug("Serving profiler on /debug/pprof")
+	logger.Debug("Serving profiler on /debug/pprof")
+
+	s.debug = &http.Server{
+		Addr:    fmt.Sprintf(":%d", s.cfg.debugPort),
+		Handler: mux,
+	}
 
-	return mux
+	return nil
 }
 
 func (s *Server) initializeGRPC() error {
@@ -251,3 +331,11 @@ func (s *Server) initializeGRPC() error {
 
 	return nil
 }
+
+func (s *Server) isGRPCEnabled() bool {
+	return s.cfg.grpcPort >= 0
+}
+
+func (s *Server) isHTTPEnabled() bool {
+	return s.cfg.httpPort >= 0
+}