Merge pull request #27 from tkashem/remote-registry

Add support to download from a remote registry

Author: openshift-merge-robot

Dockerfile

@@ -30,6 +30,7 @@ WORKDIR /registry
 COPY --from=builder /go/src/github.com/operator-framework/operator-registry/bin/initializer /bin/initializer
 COPY --from=builder /go/src/github.com/operator-framework/operator-registry/bin/registry-server /bin/registry-server
 COPY --from=builder /go/src/github.com/operator-framework/operator-registry/bin/configmap-server /bin/configmap-server
+COPY --from=builder /go/src/github.com/operator-framework/operator-registry/bin/appregistry-server /bin/appregistry-server
 COPY --from=builder /go/bin/grpc_health_probe /bin/grpc_health_probe
 
 RUN chgrp -R 0 /registry && \

appr-registry.Dockerfile

@@ -0,0 +1,28 @@
+FROM golang:1.10-alpine as builder
+
+RUN apk update && apk add sqlite build-base git mercurial
+WORKDIR /go/src/github.com/operator-framework/operator-registry
+
+COPY vendor vendor
+COPY cmd cmd
+COPY pkg pkg
+COPY Makefile Makefile
+RUN make static
+
+FROM golang:1.10-alpine as probe-builder
+
+RUN apk update && apk add build-base git
+ENV ORG github.com/grpc-ecosystem
+ENV PROJECT $ORG/grpc_health_probe
+WORKDIR /go/src/$PROJECT
+
+COPY --from=builder /go/src/github.com/operator-framework/operator-registry/vendor/$ORG/grpc-health-probe .
+COPY --from=builder /go/src/github.com/operator-framework/operator-registry/vendor .
+RUN CGO_ENABLED=0 go install -a -tags netgo -ldflags "-w"
+
+
+FROM scratch
+COPY --from=builder /go/src/github.com/operator-framework/operator-registry/bin/appregistry-server /bin/appregistry-server
+COPY --from=probe-builder /go/bin/grpc_health_probe /bin/grpc_health_probe
+EXPOSE 50051
+ENTRYPOINT ["/bin/appregistry-server"]

cmd/appregistry-server/main.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"net"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/reflection"
+
+	"github.com/operator-framework/operator-registry/pkg/api"
+	health "github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1"
+	"github.com/operator-framework/operator-registry/pkg/appregistry"
+	"github.com/operator-framework/operator-registry/pkg/lib/log"
+	"github.com/operator-framework/operator-registry/pkg/server"
+)
+
+func main() {
+	var rootCmd = &cobra.Command{
+		Short: "appregistry-server",
+		Long:  `appregistry-server downloads operator manifest(s) from remote appregistry, builds a sqlite database containing these downloaded manifest(s) and serves a grpc API to query it`,
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runCmdFunc,
+	}
+
+	rootCmd.Flags().Bool("debug", false, "enable debug logging")
+	rootCmd.Flags().StringP("kubeconfig", "k", "", "absolute path to kubeconfig file")
+	rootCmd.Flags().StringP("database", "d", "bundles.db", "name of db to output")
+	rootCmd.Flags().StringP("sources", "s", "", "comma separated list of OperatorSource object(s) {namespace}/{name}")
+	rootCmd.Flags().StringP("packages", "o", "", "comma separated list of package(s) to be downloaded from the specified operator source(s)")
+	rootCmd.Flags().StringP("port", "p", "50051", "port number to serve on")
+	rootCmd.Flags().StringP("termination-log", "t", "/dev/termination-log", "path to a container termination log file")
+	if err := rootCmd.Flags().MarkHidden("debug"); err != nil {
+		logrus.Panic(err.Error())
+	}
+
+	if err := rootCmd.Execute(); err != nil {
+		logrus.Panic(err.Error())
+	}
+}
+
+func runCmdFunc(cmd *cobra.Command, args []string) error {
+	// Immediately set up termination log
+	terminationLogPath, err := cmd.Flags().GetString("termination-log")
+	if err != nil {
+		return err
+	}
+	err = log.AddDefaultWriterHooks(terminationLogPath)
+	if err != nil {
+		return err
+	}
+	kubeconfig, err := cmd.Flags().GetString("kubeconfig")
+	if err != nil {
+		return err
+	}
+	port, err := cmd.Flags().GetString("port")
+	if err != nil {
+		return err
+	}
+	sources, err := cmd.Flags().GetString("sources")
+	if err != nil {
+		return err
+	}
+	packages, err := cmd.Flags().GetString("packages")
+	if err != nil {
+		return err
+	}
+	dbName, err := cmd.Flags().GetString("database")
+	if err != nil {
+		return err
+	}
+
+	logger := logrus.WithFields(logrus.Fields{"type": "appregistry", "port": port})
+
+	loader, err := appregistry.NewLoader(kubeconfig, logger)
+	if err != nil {
+		logger.Fatalf("error initializing - %v", err)
+	}
+
+	store, err := loader.Load(dbName, sources, packages)
+	if err != nil {
+		logger.Fatalf("error loading manifest from remote registry - %v", err)
+	}
+
+	lis, err := net.Listen("tcp", ":"+port)
+	if err != nil {
+		logger.Fatalf("failed to listen: %v", err)
+	}
+	s := grpc.NewServer()
+
+	api.RegisterRegistryServer(s, server.NewRegistryServer(store))
+	health.RegisterHealthServer(s, server.NewHealthServer())
+	reflection.Register(s)
+
+	logger.Info("serving registry")
+	if err := s.Serve(lis); err != nil {
+		logger.Fatalf("failed to serve: %v", err)
+	}
+
+	return nil
+}

go.mod

@@ -11,8 +11,11 @@ require (
 	github.com/elazarl/go-bindata-assetfs v1.0.0 // indirect
 	github.com/emicklei/go-restful-swagger12 v0.0.0-20170926063155-7524189396c6 // indirect
 	github.com/ghodss/yaml v1.0.0
+	github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9
+	github.com/go-openapi/strfmt v0.18.0
 	github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect
 	github.com/golang/lint v0.0.0-20180702182130-06c8688daad7 // indirect
+	github.com/golang/mock v1.1.1
 	github.com/golang/protobuf v1.2.0
 	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190203031600-7a902570cb17 // indirect
@@ -23,7 +26,9 @@ require (
 	github.com/jonboulle/clockwork v0.1.0 // indirect
 	github.com/mattn/go-sqlite3 v1.10.0
 	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/operator-framework/go-appr v0.0.0-20180917210448-f2aef88446f2
 	github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190125151539-1e295784b30a
+	github.com/operator-framework/operator-marketplace v0.0.0-20190212161948-a7ca81b96ad9
 	github.com/sirupsen/logrus v1.2.0
 	github.com/soheilhy/cmux v0.1.4 // indirect
 	github.com/spf13/cobra v0.0.3
@@ -38,4 +43,5 @@ require (
 	k8s.io/apiextensions-apiserver v0.0.0-20181204003618-e419c5771cdc
 	k8s.io/apimachinery v0.0.0-20190118094746-1525e4dadd2d
 	k8s.io/client-go v8.0.0+incompatible
+	sigs.k8s.io/controller-runtime v0.1.10 // indirect
 )

go.sum

@@ -74,6 +74,7 @@ github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg=
 github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
+github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -86,15 +87,13 @@ github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhp
 github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f h1:ShTPMJQes6tubcjzGMODIVG5hlrCeImaBnZzKF2N8SM=
 github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20190203031600-7a902570cb17 h1:prg2TTpTOcJF1jRWL2zSU1FQNgB0STAFNux8GK82y8k=
 github.com/gregjones/httpcache v0.0.0-20190203031600-7a902570cb17/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.5.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/grpc-ecosystem/grpc-gateway v1.6.3/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
-github.com/grpc-ecosystem/grpc-health-probe v0.2.0 h1:Os9pdioxm25j+92QbQNW7sOClH9Px9Jo9W9nhLTA5EU=
 github.com/grpc-ecosystem/grpc-health-probe v0.2.0/go.mod h1:4GVx/bTCtZaSzhjbGueDY5YgBdsmKeVx+LErv/n0L6s=
 github.com/grpc-ecosystem/grpc-health-probe v0.2.1-0.20181220223928-2bf0a5b182db h1:UxmGBzaBcWDQuQh9E1iT1dWKQFbizZ+SpTd1EL4MSqs=
 github.com/grpc-ecosystem/grpc-health-probe v0.2.1-0.20181220223928-2bf0a5b182db/go.mod h1:uBKkC2RbarFsvS5jMJHpVhTLvGlGQj9JJwkaePE3FWI=
@@ -132,10 +131,16 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.2-0.20180831124310-ae19f1b56d53/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/operator-framework/go-appr v0.0.0-20180917210448-f2aef88446f2 h1:c7gnBIMtxxenMKXZjeCuQaDfn7IGGmgh9laEGsOEeU4=
+github.com/operator-framework/go-appr v0.0.0-20180917210448-f2aef88446f2/go.mod h1:YNzwUx1i6C4dXWcffyq3yaIb0rh/K8/OvQ4vG0SNlSw=
 github.com/operator-framework/operator-lifecycle-manager v0.0.0-20181023032605-e838f7fb2186/go.mod h1:Ma5ZXd4S1vmMyewWlF7aO8CZiokR7Sd8dhSfkGkNU4U=
 github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190105193533-81104ffdc4fb/go.mod h1:XMyE4n2opUK4N6L45YGQkXXi8F9fD7XDYFv/CsS6V5I=
-github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190125151539-1e295784b30a h1:jHgwWvjTqqg5yODVyxnya5NtIdc6G3wrxc0OzzWbSh0=
+github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190125151539-1e295784b30a h1:gBTaGobFQZVx+M2ItXxJp7oxoc9dltkLMrkgyDE0Qfc=
 github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190125151539-1e295784b30a/go.mod h1:vq6TTFvg6ti1Bn6ACsZneZTmjTsURgDD6tQtVDbEgsU=
+github.com/operator-framework/operator-marketplace v0.0.0-20190208230340-d06f7b349013 h1:SyVL/xPGF6ALLDOQNchwagHz7QvESdgiiDyNoIRnMxI=
+github.com/operator-framework/operator-marketplace v0.0.0-20190208230340-d06f7b349013/go.mod h1:msZSL8pXwzQjB+hU+awVrZQw94IwJi3sNZVD3NoESIs=
+github.com/operator-framework/operator-marketplace v0.0.0-20190212161948-a7ca81b96ad9 h1:VjGYvB+9cqsf0vgO7npB1bwAIslvLFqqL1ydX9ogCRM=
+github.com/operator-framework/operator-marketplace v0.0.0-20190212161948-a7ca81b96ad9/go.mod h1:msZSL8pXwzQjB+hU+awVrZQw94IwJi3sNZVD3NoESIs=
 github.com/operator-framework/operator-registry v1.0.1/go.mod h1:1xEdZjjUg2hPEd52LG3YQ0jtwiwEGdm98S1TH5P4RAA=
 github.com/operator-framework/operator-registry v1.0.4/go.mod h1:hve6YwcjM2nGVlscLtNsp9sIIBkNZo6jlJgzWw7vP9s=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -208,7 +213,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181016170114-94acd270e44e h1:I5s8aUkxqPjgAssfOv+dVr+4/7BC40WV6JhcVoORltI=
 google.golang.org/genproto v0.0.0-20181016170114-94acd270e44e/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/grpc v1.16.0 h1:dz5IJGuC2BB7qXR5AyHNwAUBhZscK2xVez7mznh72sY=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
@@ -249,3 +253,5 @@ k8s.io/kube-openapi v0.0.0-20181031203759-72693cb1fadd h1:ggv/Vfza0i5xuhUZyYyxcc
 k8s.io/kube-openapi v0.0.0-20181031203759-72693cb1fadd/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
 k8s.io/kubernetes v1.11.7-beta.0.0.20181219023948-b875d52ea96d/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 k8s.io/kubernetes v1.11.8-beta.0.0.20190124204751-3a10094374f2/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
+sigs.k8s.io/controller-runtime v0.1.10 h1:amLOmcekVdnsD1uIpmgRqfTbQWJ2qxvQkcdeFhcotn4=
+sigs.k8s.io/controller-runtime v0.1.10/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHEHHRdJMf2jMX8=

pkg/apprclient/adapter.go

@@ -0,0 +1,77 @@
+package apprclient
+
+import (
+	"bytes"
+
+	appr "github.com/operator-framework/go-appr/appregistry"
+	apprblobs "github.com/operator-framework/go-appr/appregistry/blobs"
+	apprpackage "github.com/operator-framework/go-appr/appregistry/package_appr"
+	apprmodels "github.com/operator-framework/go-appr/models"
+)
+
+const (
+	mediaType = "helm"
+)
+
+// This interface (internal to this package) encapsulates nitty gritty details of go-appr client bindings
+type apprApiAdapter interface {
+	// ListPackages returns a list of package(s) available to the user.
+	// When namespace is specified, only package(s) associated with the given namespace are returned.
+	// If namespace is empty then visible package(s) across all namespaces are returned.
+	ListPackages(namespace string) (apprmodels.Packages, error)
+
+	// GetPackageMetadata returns metadata associated with a given package
+	GetPackageMetadata(namespace string, repository string, release string) (*apprmodels.Package, error)
+
+	// DownloadOperatorManifest downloads the blob associated with a given digest that directly corresponds to a package release
+	DownloadOperatorManifest(namespace string, repository string, digest string) ([]byte, error)
+}
+
+type apprApiAdapterImpl struct {
+	client *appr.Appregistry
+}
+
+func (a *apprApiAdapterImpl) ListPackages(namespace string) (apprmodels.Packages, error) {
+	params := apprpackage.NewListPackagesParams()
+
+	if namespace != "" {
+		params.SetNamespace(&namespace)
+	}
+
+	packages, err := a.client.PackageAppr.ListPackages(params)
+	if err != nil {
+		return nil, err
+	}
+
+	return packages.Payload, nil
+}
+
+func (a *apprApiAdapterImpl) GetPackageMetadata(namespace string, repository string, release string) (*apprmodels.Package, error) {
+	params := apprpackage.NewShowPackageParams().
+		WithNamespace(namespace).
+		WithPackage(repository).
+		WithRelease(release).
+		WithMediaType(mediaType)
+
+	pkg, err := a.client.PackageAppr.ShowPackage(params)
+	if err != nil {
+		return nil, err
+	}
+
+	return pkg.Payload, nil
+}
+
+func (a *apprApiAdapterImpl) DownloadOperatorManifest(namespace string, repository string, digest string) ([]byte, error) {
+	params := apprblobs.NewPullBlobParams().
+		WithNamespace(namespace).
+		WithPackage(repository).
+		WithDigest(digest)
+
+	writer := &bytes.Buffer{}
+	_, err := a.client.Blobs.PullBlob(params, writer)
+	if err != nil {
+		return nil, err
+	}
+
+	return writer.Bytes(), nil
+}

pkg/apprclient/apprclient.go

@@ -0,0 +1,38 @@
+package apprclient
+
+import (
+	"net/url"
+
+	"github.com/go-openapi/runtime"
+	httptransport "github.com/go-openapi/runtime/client"
+	"github.com/go-openapi/strfmt"
+	apprclient "github.com/operator-framework/go-appr/appregistry"
+)
+
+// NewClientFactory return a factory which can be used to instantiate a new appregistry client
+func NewClientFactory() ClientFactory {
+	return &factory{}
+}
+
+type ClientFactory interface {
+	// New returns a new instance of appregistry Client from given source and type
+	New(sourceType, source string) (Client, error)
+}
+
+type factory struct{}
+
+func (f *factory) New(sourceType, source string) (Client, error) {
+	u, err := url.Parse(source)
+	if err != nil {
+		return nil, err
+	}
+
+	transport := httptransport.New(u.Host, u.Path, []string{u.Scheme})
+	transport.Consumers["application/x-gzip"] = runtime.ByteStreamConsumer()
+	c := apprclient.New(transport, strfmt.Default)
+
+	return &client{
+		adapter: &apprApiAdapterImpl{client: c},
+		decoder: &blobDecoderImpl{},
+	}, nil
+}