use buf to generate networking code

.github/workflows/build-and-push.yaml

@@ -47,7 +47,7 @@ jobs:
         # This is committed at generated/ but building here ensure the make
         # targets do not go stale and that any updates are committed.
         run: |
-          make proto-generated-srcs
+          make buf-install buf-generated-srcs
           git diff --exit-code HEAD
 
       - name: Go Unit Test

.gitignore

@@ -22,3 +22,4 @@ tags
 nohup.out
 test/mocks
 __debug_bin
+.DS_Store

CHANGELOG.md

@@ -6,6 +6,8 @@ Please avoid adding duplicate information across this changelog and JIRA/doc inp
 
 ## [NEXT RELEASE]
 
+- gRPC code is now generated with [buf](https://github.com/bufbuild/buf).
+
 ## [0.7.1]
 
 - ROX-15237: The Openshift 4 Demo flavor now supports testing of unreleased versions.

Makefile

@@ -93,12 +93,12 @@ clean-image:
 #############
 
 .PHONY: unit-test
-unit-test: proto-generated-srcs
+unit-test: buf-generated-srcs
 	@echo "+ $@"
 	@go test -v ./...
 
 .PHONY: go-e2e-tests
-go-e2e-tests: proto-generated-srcs
+go-e2e-tests: buf-install buf-generated-srcs
 	@kubectl apply -f workflows/
 	@go test ./test/e2e/... -tags=e2e -v -parallel 5 -count 1 -cover -timeout 1h
 
@@ -124,87 +124,17 @@ e2e-tests:
 ##############
 ## Protobuf ##
 ##############
-# Tool versions.
-protoc-version = 3.11.2
-protoc-gen-go-version = 1.3.2
-protoc-gen-grpc-gateway-version = 1.12.1
-protoc-gen-swagger-version = 1.12.1
-
-# Tool binary paths
-protoc = $(GOPATH)/bin/protoc
-protoc-gen-go = $(GOPATH)/bin/protoc-gen-go
-protoc-gen-grpc-gateway = $(GOPATH)/bin/protoc-gen-grpc-gateway
-protoc-gen-swagger = $(GOPATH)/bin/protoc-gen-swagger
-
-# The protoc zip url changes depending on if we're running in CI or not.
-ifeq ($(shell uname -s),Linux)
-PROTOC_ZIP = https://github.com/protocolbuffers/protobuf/releases/download/v$(protoc-version)/protoc-$(protoc-version)-linux-x86_64.zip
-endif
-ifeq ($(shell uname -s),Darwin)
-PROTOC_ZIP = https://github.com/protocolbuffers/protobuf/releases/download/v$(protoc-version)/protoc-$(protoc-version)-osx-x86_64.zip
-endif
-
-# This target installs the protoc binary.
-$(protoc):
-	@echo "+ $@"
-	@echo "Installing protoc $(protoc-version) to $(protoc)"
-	@mkdir -p $(GOPATH)/bin
-	@wget -q $(PROTOC_ZIP) -O /tmp/protoc.zip
-	@unzip -o -q -d /tmp /tmp/protoc.zip bin/protoc
-	@install /tmp/bin/protoc $(protoc)
-
-# This target installs the protoc-gen-go binary.
-$(protoc-gen-go):
-	@echo "+ $@"
-	@echo "Installing protoc-gen-go $(protoc-gen-go-version) to $(protoc-gen-go)"
-	@cd /tmp; go install github.com/golang/protobuf/protoc-gen-go@v$(protoc-gen-go-version)
-
-# This target installs the protoc-gen-grpc-gateway binary.
-$(protoc-gen-grpc-gateway):
-	@echo "+ $@"
-	@echo "Installing protoc-gen-grpc-gateway $(protoc-gen-grpc-gateway-version) to $(protoc-gen-grpc-gateway)"
-	@cd /tmp; go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@v$(protoc-gen-grpc-gateway-version)
-
-# This target installs the protoc-gen-swagger binary.
-$(protoc-gen-swagger):
-	@echo "+ $@"
-	@echo "Installing protoc-gen-swagger $(protoc-gen-swagger-version) to $(protoc-gen-swagger)"
-	@cd /tmp; go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger@v$(protoc-gen-swagger-version)
-
-# This target installs all of the protoc related binaries.
-.PHONY: protoc-tools
-protoc-tools: $(protoc) $(protoc-gen-go) $(protoc-gen-grpc-gateway) $(protoc-gen-swagger)
-
-PROTO_INPUT_DIR   = proto/api/v1
-PROTO_THIRD_PARTY = proto/third_party
-PROTO_FILES       = service.proto
-PROTO_OUTPUT_DIR  = generated/api/v1
+# This target installs the buf binary.
+buf-install:
+	@./scripts/install-buf.sh
 
 # This target compiles proto files into:
 # - Go gRPC bindings
 # - Go gRPC-Gateway bindings
 # - JSON Swagger definitions file
-.PHONY: proto-generated-srcs
-proto-generated-srcs: protoc-tools
-	@echo "+ $@"
-	@mkdir -p $(PROTO_OUTPUT_DIR)
-	# Generate gRPC bindings
-	$(protoc) -I$(PROTO_INPUT_DIR) \
-		-I$(PROTO_THIRD_PARTY) \
-		--go_out=plugins=grpc:$(PROTO_OUTPUT_DIR) \
-		$(PROTO_FILES)
-
-	# Generate gRPC-Gateway bindings
-	$(protoc) -I$(PROTO_INPUT_DIR) \
-		-I$(PROTO_THIRD_PARTY) \
-		--grpc-gateway_out=logtostderr=true:$(PROTO_OUTPUT_DIR) \
-		$(PROTO_FILES)
-
-	# Generate JSON Swagger manifest
-	$(protoc) -I$(PROTO_INPUT_DIR) \
-		-I$(PROTO_THIRD_PARTY) \
-		--swagger_out=logtostderr=true:$(PROTO_OUTPUT_DIR) \
-		$(PROTO_FILES)
+.PHONY: buf-generated-srcs
+buf-generated-srcs:
+	@buf generate
 
 ##########
 ## Kube ##

README.md

@@ -16,13 +16,15 @@ toolchain, it is also possible to rely on CI. CI will lint, build and push the
 infra server. And then deploy it to a development cluster created using the
 production infra deployment. A
 (comment)[https://github.com/stackrox/infra/pull/711#issuecomment-1270457578]
-will appear on PRs with more detail. 
+will appear on PRs with more detail.
 
 ### Regenerate Go bindings from protos
 
 To regenerate the Go proto bindings, run:
 
-`make proto-generated-srcs`
+`make buf-generated-srcs`
+
+Note: You need to have `buf` installed for this. Do this e.g. with `make buf-install`.
 
 ### Building the server and cli
 

auth/handlers.go

@@ -7,7 +7,7 @@ import (
 	"net/http"
 
 	"github.com/coreos/go-oidc/v3/oidc"
-	v1 "github.com/stackrox/infra/generated/api/v1"
+	v1 "github.com/stackrox/infra/generated/proto/api/v1"
 	"golang.org/x/oauth2"
 )
 
@@ -35,13 +35,13 @@ func (a OidcAuth) ValidateUser(token string) (*v1.User, error) {
 
 // GenerateServiceAccountToken generates a service account JWT containing a
 // v1.User struct.
-func (a OidcAuth) GenerateServiceAccountToken(svcacct v1.ServiceAccount) (string, error) {
+func (a OidcAuth) GenerateServiceAccountToken(svcacct *v1.ServiceAccount) (string, error) {
 	return a.jwtSvcAcct.Generate(svcacct)
 }
 
 // ValidateServiceAccountToken validates a service account JWT and returns the
 // contained v1.ServiceAccount struct.
-func (a OidcAuth) ValidateServiceAccountToken(token string) (v1.ServiceAccount, error) {
+func (a OidcAuth) ValidateServiceAccountToken(token string) (*v1.ServiceAccount, error) {
 	return a.jwtSvcAcct.Validate(token)
 }
 
@@ -56,7 +56,7 @@ func (a OidcAuth) loginHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	oauth2.SetAuthURLParam("response_mode", "query")
+	_ = oauth2.SetAuthURLParam("response_mode", "query")
 	redirectURL := a.conf.AuthCodeURL(stateToken)
 
 	// Redirect to authorization server so that the user can login externally.

auth/tokenizer.go

@@ -10,7 +10,7 @@ import (
 	"github.com/golang/protobuf/ptypes/timestamp"
 	"github.com/pkg/errors"
 	"github.com/stackrox/infra/auth/claimrule"
-	v1 "github.com/stackrox/infra/generated/api/v1"
+	v1 "github.com/stackrox/infra/generated/proto/api/v1"
 	"golang.org/x/oauth2"
 )
 
@@ -186,15 +186,15 @@ func NewUserTokenizer(lifetime time.Duration, secret string) *userTokenizer {
 // userClaims facilitates the arshalling/unmarshalling of JWTs containing v1
 // .User data.
 type userClaims struct {
-	User v1.User `json:"user"`
+	User *v1.User `json:"user"`
 	jwt.StandardClaims
 }
 
 // Generate generates a user JWT containing a v1.User struct.
 func (t userTokenizer) Generate(user *v1.User) (string, error) {
 	now := time.Now()
 	claims := userClaims{
-		User: *user,
+		User: user,
 		StandardClaims: jwt.StandardClaims{
 			ExpiresAt: now.Add(t.lifetime).Unix(),
 			NotBefore: now.Unix(),
@@ -217,12 +217,14 @@ func (t userTokenizer) Validate(token string) (*v1.User, error) {
 	}); err != nil {
 		return nil, err
 	}
-	return &claims.User, nil
+	return claims.User, nil
 }
 
-type serviceAccountValidator v1.ServiceAccount
+type serviceAccountValidator struct {
+	*v1.ServiceAccount
+}
 
-func (s serviceAccountValidator) Valid() error {
+func (s *serviceAccountValidator) Valid() error {
 	_, isExcluded := excludedEmails[s.Email]
 	if isExcluded {
 		return errors.New("email address is excluded")
@@ -254,38 +256,38 @@ type serviceAccountTokenizer struct {
 }
 
 // Generate generates a service account JWT containing a v1.ServiceAccount.
-func (t serviceAccountTokenizer) Generate(svcacct v1.ServiceAccount) (string, error) {
+func (t serviceAccountTokenizer) Generate(svcacct *v1.ServiceAccount) (string, error) {
 	// Set issuing and expiration times on new ServiceAccount.
 	now := time.Now()
 	svcacct.ExpiresAt = now.Add(t.lifetime).Unix()
 	svcacct.NotBefore = now.Unix()
 	svcacct.IssuedAt = now.Unix()
 
-	svc := serviceAccountValidator(svcacct)
+	svc := serviceAccountValidator{svcacct}
 
 	// Ensure that our service account is well-formed.
 	if err := svc.Valid(); err != nil {
 		return "", errors.Wrap(err, "invalid service account")
 	}
 
 	// Generate new token object, containing the wrapped data.
-	token := jwt.NewWithClaims(jwt.SigningMethodHS256, svc)
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, &svc)
 
 	// Sign and get the complete encoded token as a string using the secret
 	return token.SignedString(t.secret)
 }
 
 // Validate validates a service account JWT and returns the contained
 // v1.ServiceAccount.
-func (t serviceAccountTokenizer) Validate(token string) (v1.ServiceAccount, error) {
+func (t serviceAccountTokenizer) Validate(token string) (*v1.ServiceAccount, error) {
 	var claims serviceAccountValidator
 	if _, err := jwt.ParseWithClaims(token, &claims, func(_ *jwt.Token) (interface{}, error) {
 		return t.secret, nil
 	}); err != nil {
-		return v1.ServiceAccount{}, err
+		return &v1.ServiceAccount{}, err
 	}
 
-	return v1.ServiceAccount(claims), nil
+	return claims.ServiceAccount, nil
 }
 
 // accessTokenizer facilitates the verification of access roles generated by

buf.gen.yaml

@@ -0,0 +1,17 @@
+version: v1
+plugins:
+  - remote: buf.build/library/plugins/go:v1.27.1-1
+    out: generated
+    opt:
+      - paths=source_relative
+  - remote: buf.build/library/plugins/go-grpc:v1.1.0-2
+    out: generated
+    opt:
+      - paths=source_relative
+      - require_unimplemented_servers=false
+  - remote: buf.build/grpc-ecosystem/plugins/grpc-gateway:v2.6.0-1
+    out: generated
+    opt:
+      - paths=source_relative
+  - remote: buf.build/grpc-ecosystem/plugins/openapiv2:v2.6.0-1
+    out: generated

buf.lock

@@ -0,0 +1,7 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: googleapis
+    repository: googleapis
+    commit: 5ae7f88519b04fe1965da0f8a375a088

buf.yaml

@@ -0,0 +1,9 @@
+version: v1
+breaking:
+  use:
+    - FILE
+lint:
+  use:
+    - DEFAULT
+deps:
+  - buf.build/googleapis/googleapis

cmd/infractl/cli/upgrade/command.go

@@ -11,7 +11,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/stackrox/infra/cmd/infractl/common"
-	v1 "github.com/stackrox/infra/generated/api/v1"
+	v1 "github.com/stackrox/infra/generated/proto/api/v1"
 	"github.com/stackrox/infra/pkg/platform"
 	"google.golang.org/grpc"
 )

cmd/infractl/cluster/artifacts/command.go

@@ -15,7 +15,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/stackrox/infra/cmd/infractl/common"
-	v1 "github.com/stackrox/infra/generated/api/v1"
+	v1 "github.com/stackrox/infra/generated/proto/api/v1"
 	"google.golang.org/grpc"
 )
 
@@ -65,25 +65,25 @@ func DownloadArtifacts(ctx context.Context, client v1.ClusterServiceClient, id s
 	// If no --download-dir flag was given, skip downloading the artifacts
 	// altogether.
 	if downloadDir == "" {
-		return prettyClusterArtifacts(*resp), nil
+		return prettyClusterArtifacts{resp}, nil
 	}
 
 	for _, artifact := range resp.Artifacts {
-		filename, err := download(downloadDir, *artifact)
+		filename, err := download(downloadDir, artifact)
 		if err != nil {
 			return nil, err
 		}
 		if strings.HasSuffix(filename, ".tgz") {
-			unpackSingleArtifact(filename, downloadDir, *artifact)
+			unpackSingleArtifact(filename, downloadDir, artifact)
 		}
 	}
 
-	return prettyClusterArtifacts(*resp), nil
+	return prettyClusterArtifacts{resp}, nil
 }
 
 // download will save the given cluster artifact to disk inside the given
 // directory.
-func download(downloadDir string, artifact v1.Artifact) (filename string, err error) {
+func download(downloadDir string, artifact *v1.Artifact) (filename string, err error) {
 	// Create the download directory if it doesn't exist. All artifacts will be
 	// downloaded into this directory.
 	if err := os.MkdirAll(downloadDir, 0755); err != nil {
@@ -134,7 +134,7 @@ func download(downloadDir string, artifact v1.Artifact) (filename string, err er
 // ignored here as the artifact is already saved.
 //
 //nolint:errcheck
-func unpackSingleArtifact(tgzFilename string, downloadDir string, artifact v1.Artifact) {
+func unpackSingleArtifact(tgzFilename string, downloadDir string, artifact *v1.Artifact) {
 	file, err := os.Open(tgzFilename)
 	if err != nil {
 		return

cmd/infractl/cluster/artifacts/fancy.go

@@ -5,10 +5,12 @@ import (
 
 	"github.com/spf13/cobra"
 
-	v1 "github.com/stackrox/infra/generated/api/v1"
+	v1 "github.com/stackrox/infra/generated/proto/api/v1"
 )
 
-type prettyClusterArtifacts v1.ClusterArtifacts
+type prettyClusterArtifacts struct {
+	*v1.ClusterArtifacts
+}
 
 func (p prettyClusterArtifacts) PrettyPrint(cmd *cobra.Command) {
 	for _, artifact := range p.Artifacts {