Skip to content

[ws-manager] Support image-builder on ws-manager gRPC connection #9335

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 19, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions components/ws-manager-api/go/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ type ServiceConfiguration struct {
} `json:"tls"`
RateLimits map[string]grpc.RateLimit `json:"ratelimits"`
} `json:"rpcServer"`
ImageBuilderProxy struct {
TargetAddr string `json:"targetAddr"`
} `json:"imageBuilderProxy"`

PProf struct {
Addr string `json:"addr"`
Expand Down
27 changes: 27 additions & 0 deletions components/ws-manager/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,21 @@ package cmd
import (
"context"
"net"
"strings"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"

"github.com/bombsimon/logrusr/v2"
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
"github.com/mwitkow/grpc-proxy/proxy"
"github.com/spf13/cobra"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -138,6 +143,8 @@ var runCmd = &cobra.Command{
log.Warn("no TLS configured - gRPC server will be unsecured")
}

grpcOpts = append(grpcOpts, grpc.UnknownServiceHandler(proxy.TransparentHandler(imagebuilderDirector(cfg.ImageBuilderProxy.TargetAddr))))

grpcServer := grpc.NewServer(grpcOpts...)
defer grpcServer.Stop()
grpc_prometheus.Register(grpcServer)
Expand Down Expand Up @@ -200,3 +207,23 @@ func init() {
var (
scheme = runtime.NewScheme()
)

func imagebuilderDirector(targetAddr string) proxy.StreamDirector {
if targetAddr == "" {
return func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) {
return ctx, nil, status.Error(codes.Unimplemented, "Unknown method")
}
}

return func(ctx context.Context, fullMethodName string) (outCtx context.Context, conn *grpc.ClientConn, err error) {
md, _ := metadata.FromIncomingContext(ctx)
outCtx = metadata.NewOutgoingContext(ctx, md.Copy())

if strings.HasPrefix(fullMethodName, "/builder.") {
conn, err = grpc.DialContext(ctx, targetAddr, grpc.WithInsecure())
return
}

return outCtx, nil, status.Error(codes.Unimplemented, "Unknown method")
}
}
3 changes: 3 additions & 0 deletions components/ws-manager/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ require (
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/Microsoft/hcsshim v0.8.17 // indirect
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
github.com/beorn7/perks v1.0.1 // indirect
Expand Down Expand Up @@ -81,6 +82,7 @@ require (
github.com/moby/sys/mountinfo v0.4.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mwitkow/grpc-proxy v0.0.0-20220126150247-db34e7bfee32 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
Expand Down Expand Up @@ -111,6 +113,7 @@ require (
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
honnef.co/go/tools v0.1.3 // indirect
k8s.io/apiextensions-apiserver v0.23.4 // indirect
k8s.io/component-base v0.23.4 // indirect
k8s.io/klog/v2 v2.30.0 // indirect
Expand Down
8 changes: 8 additions & 0 deletions components/ws-manager/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z
github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgRRqzCuPshRkQ7I=
Expand Down Expand Up @@ -590,6 +591,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/grpc-proxy v0.0.0-20220126150247-db34e7bfee32 h1:CC9KzU7WPrK6DTppkUGiwmttoHCNwOLT7Z+stp1eIpU=
github.com/mwitkow/grpc-proxy v0.0.0-20220126150247-db34e7bfee32/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
Expand Down Expand Up @@ -938,6 +941,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
Expand Down Expand Up @@ -1050,6 +1054,7 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down Expand Up @@ -1232,6 +1237,7 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210401141331-865547bb08e2/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
google.golang.org/genproto v0.0.0-20210420162539-3c870d7478d2/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
Expand Down Expand Up @@ -1334,6 +1340,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o=
honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
k8s.io/api v0.23.4 h1:85gnfXQOWbJa1SiWGpE9EEtHs0UVvDyIsSMpEtl2D4E=
k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI=
k8s.io/apiextensions-apiserver v0.23.4 h1:AFDUEu/yEf0YnuZhqhIFhPLPhhcQQVuR1u3WCh0rveU=
Expand Down
101 changes: 71 additions & 30 deletions dev/gpctl/cmd/imagebuilds.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,12 @@ package cmd

import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net"
"path/filepath"

"github.com/spf13/cobra"
"golang.org/x/xerrors"
Expand All @@ -29,56 +33,93 @@ var imagebuildsCmd = &cobra.Command{
func init() {
imagebuildsCmd.PersistentFlags().StringP("tls", "t", "", "TLS certificate when connecting to a secured gRPC endpoint")
imagebuildsCmd.PersistentFlags().Bool("mk3", true, "use image-builder mk3")
imagebuildsCmd.PersistentFlags().String("host", "", "dial a host directly")
imagebuildsCmd.PersistentFlags().String("tls-path", "", "TLS certificate when connecting to a secured gRPC endpoint")

rootCmd.AddCommand(imagebuildsCmd)
}

func getImagebuildsClient(ctx context.Context) (*grpc.ClientConn, api.ImageBuilderClient, error) {
cfg, namespace, err := getKubeconfig()
if err != nil {
return nil, nil, err
}
clientSet, err := kubernetes.NewForConfig(cfg)
if err != nil {
return nil, nil, err
}
host, _ := imagebuildsCmd.PersistentFlags().GetString("host")
if host == "" {
cfg, namespace, err := getKubeconfig()
if err != nil {
return nil, nil, err
}
clientSet, err := kubernetes.NewForConfig(cfg)
if err != nil {
return nil, nil, err
}

comp := "image-builder"
if mk3, _ := imagebuildsCmd.PersistentFlags().GetBool("mk3"); mk3 {
comp = "image-builder-mk3"
}
comp := "image-builder"
if mk3, _ := imagebuildsCmd.PersistentFlags().GetBool("mk3"); mk3 {
comp = "image-builder-mk3"
}

freePort, err := GetFreePort()
if err != nil {
return nil, nil, err
}
freePort, err := GetFreePort()
if err != nil {
return nil, nil, err
}

port := fmt.Sprintf("%d:8080", freePort)
podName, err := util.FindAnyPodForComponent(clientSet, namespace, comp)
if err != nil {
return nil, nil, err
}
readychan, errchan := util.ForwardPort(ctx, cfg, namespace, podName, port)
select {
case <-readychan:
case err := <-errchan:
return nil, nil, err
case <-ctx.Done():
return nil, nil, ctx.Err()
port := fmt.Sprintf("%d:8080", freePort)
podName, err := util.FindAnyPodForComponent(clientSet, namespace, comp)
if err != nil {
return nil, nil, err
}
readychan, errchan := util.ForwardPort(ctx, cfg, namespace, podName, port)
select {
case <-readychan:
case err := <-errchan:
return nil, nil, err
case <-ctx.Done():
return nil, nil, ctx.Err()
}
host = fmt.Sprintf("localhost:%d", freePort)
}

secopt := grpc.WithInsecure()
cert, _ := workspacesCmd.Flags().GetString("tls")
cert, _ := imagebuildsCmd.Flags().GetString("tls")
if cert != "" {
creds, err := credentials.NewClientTLSFromFile(cert, "")
if err != nil {
return nil, nil, xerrors.Errorf("could not load tls cert: %w", err)
}

secopt = grpc.WithTransportCredentials(creds)
} else if fn, _ := imagebuildsCmd.Flags().GetString("tls-path"); fn != "" {
crt, err := ioutil.ReadFile(filepath.Join(fn, "tls.crt"))
if err != nil {
return nil, nil, err
}
key, err := ioutil.ReadFile(filepath.Join(fn, "tls.key"))
if err != nil {
return nil, nil, err
}
cert, err := tls.X509KeyPair(crt, key)
if err != nil {
return nil, nil, err
}

ca, err := ioutil.ReadFile(filepath.Join(fn, "ca.crt"))
if err != nil {
return nil, nil, err
}
certPool := x509.NewCertPool()
certPool.AppendCertsFromPEM(ca)

creds := credentials.NewTLS(&tls.Config{
Certificates: []tls.Certificate{cert},
RootCAs: certPool,
ServerName: "ws-manager",
})
if err != nil {
return nil, nil, xerrors.Errorf("could not load tls cert: %w", err)
}

secopt = grpc.WithTransportCredentials(creds)
}

conn, err := grpc.Dial(fmt.Sprintf("localhost:%d", freePort), secopt, util.WithClientUnaryInterceptor())
conn, err := grpc.Dial(host, secopt, util.WithClientUnaryInterceptor())
if err != nil {
return nil, nil, err
}
Expand Down
2 changes: 2 additions & 0 deletions install/installer/pkg/common/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ const (
WSManagerComponent = "ws-manager"
WSManagerBridgeComponent = "ws-manager-bridge"
WSProxyComponent = "ws-proxy"
ImageBuilderComponent = "image-builder-mk3"
ImageBuilderRPCPort = 8080

AnnotationConfigChecksum = "gitpod.io/checksum_config"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"github.com/gitpod-io/gitpod/installer/pkg/components/dashboard"
"github.com/gitpod-io/gitpod/installer/pkg/components/database"
ide_proxy "github.com/gitpod-io/gitpod/installer/pkg/components/ide-proxy"
imagebuildermk3 "github.com/gitpod-io/gitpod/installer/pkg/components/image-builder-mk3"
"github.com/gitpod-io/gitpod/installer/pkg/components/migrations"
"github.com/gitpod-io/gitpod/installer/pkg/components/minio"
openvsxproxy "github.com/gitpod-io/gitpod/installer/pkg/components/openvsx-proxy"
Expand All @@ -26,7 +25,6 @@ var Objects = common.CompositeRenderFunc(
dashboard.Objects,
database.Objects,
ide_proxy.Objects,
imagebuildermk3.Objects,
migrations.Objects,
minio.Objects,
openvsxproxy.Objects,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/gitpod-io/gitpod/installer/pkg/common"
agentsmith "github.com/gitpod-io/gitpod/installer/pkg/components/agent-smith"
"github.com/gitpod-io/gitpod/installer/pkg/components/blobserve"
imagebuildermk3 "github.com/gitpod-io/gitpod/installer/pkg/components/image-builder-mk3"
registryfacade "github.com/gitpod-io/gitpod/installer/pkg/components/registry-facade"
"github.com/gitpod-io/gitpod/installer/pkg/components/workspace"
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
Expand All @@ -23,6 +24,7 @@ var Objects = common.CompositeRenderFunc(
wsdaemon.Objects,
wsmanager.Objects,
wsproxy.Objects,
imagebuildermk3.Objects,
)

var Helm = common.CompositeHelmFunc()
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@

package image_builder_mk3

import "github.com/gitpod-io/gitpod/installer/pkg/common"

const (
PullSecretFile = "/config/pull-secret.json"
BuilderImage = "image-builder-mk3/bob"
Component = "image-builder-mk3"
RPCPort = 8080
Component = common.ImageBuilderComponent
RPCPort = common.ImageBuilderRPCPort
RPCPortName = "service"
PProfPort = 6060
PrometheusPort = 9500
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,26 @@ func networkpolicy(ctx *common.RenderContext) ([]runtime.Object, error) {
Spec: networkingv1.NetworkPolicySpec{
PodSelector: metav1.LabelSelector{MatchLabels: labels},
PolicyTypes: []networkingv1.PolicyType{"Ingress"},
Ingress: []networkingv1.NetworkPolicyIngressRule{{
From: []networkingv1.NetworkPolicyPeer{{
PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{
"component": server.Component,
}},
}},
}},
Ingress: []networkingv1.NetworkPolicyIngressRule{
{
From: []networkingv1.NetworkPolicyPeer{
{
PodSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"component": server.Component,
},
},
},
{
PodSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"component": common.WSManagerComponent,
},
},
},
},
},
},
},
},
}, nil
Expand Down
5 changes: 5 additions & 0 deletions install/installer/pkg/components/ws-manager/configmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,11 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
},
RateLimits: map[string]grpc.RateLimit{}, // todo(sje) add values
},
ImageBuilderProxy: struct {
TargetAddr string "json:\"targetAddr\""
}{
TargetAddr: fmt.Sprintf("%s.%s.svc.cluster.local:%d", common.ImageBuilderComponent, ctx.Namespace, common.ImageBuilderRPCPort),
},
PProf: struct {
Addr string `json:"addr"`
}{Addr: "localhost:6060"},
Expand Down