Skip to content

[ws-manager] Explicitly proxy image-builder #9714

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 10, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions components/ws-manager/BUILD.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ packages:
- components/content-service-api/go:lib
- components/content-service:lib
- components/registry-facade-api/go:lib
- components/image-builder-api/go:lib
- components/ws-daemon-api/go:lib
- components/ws-manager-api/go:lib
env:
Expand Down
39 changes: 12 additions & 27 deletions components/ws-manager/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,16 @@ package cmd
import (
"context"
"net"
"strings"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"

"github.com/bombsimon/logrusr/v2"
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
"github.com/mwitkow/grpc-proxy/proxy"
"github.com/spf13/cobra"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes"
Expand All @@ -34,7 +29,9 @@ import (
"github.com/gitpod-io/gitpod/common-go/log"
"github.com/gitpod-io/gitpod/common-go/pprof"
"github.com/gitpod-io/gitpod/content-service/pkg/layer"
imgbldr "github.com/gitpod-io/gitpod/image-builder/api"
"github.com/gitpod-io/gitpod/ws-manager/pkg/manager"
"github.com/gitpod-io/gitpod/ws-manager/pkg/proxy"
)

// serveCmd represents the serve command
Expand Down Expand Up @@ -144,12 +141,20 @@ var runCmd = &cobra.Command{
log.Warn("no TLS configured - gRPC server will be unsecured")
}

grpcOpts = append(grpcOpts, grpc.UnknownServiceHandler(proxy.TransparentHandler(imagebuilderDirector(cfg.ImageBuilderProxy.TargetAddr))))

grpcServer := grpc.NewServer(grpcOpts...)
defer grpcServer.Stop()
grpc_prometheus.Register(grpcServer)

if cfg.ImageBuilderProxy.TargetAddr != "" {
// Note: never use block here, because image-builder connects to ws-manager,
// and if we blocked here, ws-manager wouldn't come up, hence we couldn't connect to ws-manager.
conn, err := grpc.Dial(cfg.ImageBuilderProxy.TargetAddr, grpc.WithInsecure())
if err != nil {
log.WithError(err).Fatal("failed to connect to image builder")
}
imgbldr.RegisterImageBuilderServer(grpcServer, proxy.ImageBuilder{D: imgbldr.NewImageBuilderClient(conn)})
}

manager.Register(grpcServer, mgmt)
lis, err := net.Listen("tcp", cfg.RPCServer.Addr)
if err != nil {
Expand Down Expand Up @@ -208,23 +213,3 @@ func init() {
var (
scheme = runtime.NewScheme()
)

func imagebuilderDirector(targetAddr string) proxy.StreamDirector {
if targetAddr == "" {
return func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) {
return ctx, nil, status.Error(codes.Unimplemented, "Unknown method")
}
}

return func(ctx context.Context, fullMethodName string) (outCtx context.Context, conn *grpc.ClientConn, err error) {
md, _ := metadata.FromIncomingContext(ctx)
outCtx = metadata.NewOutgoingContext(ctx, md.Copy())

if strings.HasPrefix(fullMethodName, "/builder.") {
conn, err = grpc.DialContext(ctx, targetAddr, grpc.WithInsecure())
return
}

return outCtx, nil, status.Error(codes.Unimplemented, "Unknown method")
}
}
4 changes: 3 additions & 1 deletion components/ws-manager/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/content-service v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/content-service/api v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/image-builder/api v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/registry-facade/api v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/ws-daemon/api v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/ws-manager/api v0.0.0-00010101000000-000000000000
Expand All @@ -19,7 +20,6 @@ require (
github.com/google/uuid v1.2.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/imdario/mergo v0.3.12
github.com/mwitkow/grpc-proxy v0.0.0-20220126150247-db34e7bfee32
github.com/opentracing/opentracing-go v1.2.0
github.com/prometheus/client_golang v1.12.1
github.com/sirupsen/logrus v1.8.1
Expand Down Expand Up @@ -133,6 +133,8 @@ replace github.com/gitpod-io/gitpod/content-service => ../content-service // lee

replace github.com/gitpod-io/gitpod/content-service/api => ../content-service-api/go // leeway

replace github.com/gitpod-io/gitpod/image-builder/api => ../image-builder-api/go // leeway

replace github.com/gitpod-io/gitpod/registry-facade/api => ../registry-facade-api/go // leeway

replace github.com/gitpod-io/gitpod/ws-daemon/api => ../ws-daemon-api/go // leeway
Expand Down
6 changes: 0 additions & 6 deletions components/ws-manager/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -630,8 +630,6 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/grpc-proxy v0.0.0-20220126150247-db34e7bfee32 h1:CC9KzU7WPrK6DTppkUGiwmttoHCNwOLT7Z+stp1eIpU=
github.com/mwitkow/grpc-proxy v0.0.0-20220126150247-db34e7bfee32/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
Expand Down Expand Up @@ -1000,7 +998,6 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
Expand Down Expand Up @@ -1114,7 +1111,6 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down Expand Up @@ -1306,7 +1302,6 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210401141331-865547bb08e2/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
google.golang.org/genproto v0.0.0-20210420162539-3c870d7478d2/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
Expand Down Expand Up @@ -1409,7 +1404,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA=
k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI=
Expand Down
77 changes: 77 additions & 0 deletions components/ws-manager/pkg/proxy/imagebuilder.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information.

package proxy

import (
"context"

"github.com/gitpod-io/gitpod/image-builder/api"
"google.golang.org/protobuf/proto"
)

type ImageBuilder struct {
D api.ImageBuilderClient

api.UnimplementedImageBuilderServer
}

func (p ImageBuilder) ResolveBaseImage(ctx context.Context, req *api.ResolveBaseImageRequest) (*api.ResolveBaseImageResponse, error) {
return p.D.ResolveBaseImage(ctx, req)
}

func (p ImageBuilder) ResolveWorkspaceImage(ctx context.Context, req *api.ResolveWorkspaceImageRequest) (*api.ResolveWorkspaceImageResponse, error) {
return p.D.ResolveWorkspaceImage(ctx, req)
}

func (p ImageBuilder) Build(req *api.BuildRequest, srv api.ImageBuilder_BuildServer) error {
c, err := p.D.Build(srv.Context(), req)
if err != nil {
return err
}
defer c.CloseSend()

return forwardStream(srv.Context(), c.Recv, srv.Send)
}

func (p ImageBuilder) Logs(req *api.LogsRequest, srv api.ImageBuilder_LogsServer) error {
c, err := p.D.Logs(srv.Context(), req)
if err != nil {
return err
}
defer c.CloseSend()

return forwardStream(srv.Context(), c.Recv, srv.Send)
}

func (p ImageBuilder) ListBuilds(ctx context.Context, req *api.ListBuildsRequest) (*api.ListBuildsResponse, error) {
return p.D.ListBuilds(ctx, req)
}

type ProtoMessage interface {
proto.Message
comparable
}

func forwardStream[R ProtoMessage](ctx context.Context, recv func() (R, error), send func(R) error) error {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🏅

for {
resp, err := recv()
if err != nil {
return err
}

// generic hack, can't compare R to nil because R's default value is unclear (not even sure this is nil)
// But, we can get the default value which will be nil because underneath R is an interface.
var defaultResp R
if resp == defaultResp {
break
}
err = send(resp)
if err != nil {
return err
}
}

return nil
}