[image-builder] Copy image rather than use buildkit

To work around:
containerd/containerd#5978

Author: csweichel

components/image-builder-bob/pkg/builder/builder.go

@@ -6,7 +6,10 @@ package builder
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -17,14 +20,13 @@ import (
 	"github.com/gitpod-io/gitpod/common-go/log"
 
 	"github.com/containerd/console"
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/moby/buildkit/client"
-	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/session"
-	"github.com/moby/buildkit/util/contentutil"
-	"github.com/moby/buildkit/util/imageutil"
 	"github.com/moby/buildkit/util/progress/progressui"
+	"github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
@@ -193,20 +195,15 @@ func (b *Builder) buildBaseLayer(ctx context.Context, cl *client.Client) error {
 }
 
 func (b *Builder) buildWorkspaceImage(ctx context.Context, cl *client.Client) (err error) {
-	// Note: buildkit does not handle/export image config by default. That's why we need
-	//       to download it ourselves and explicitely export it.
-	//       See https://github.com/moby/buildkit/issues/2362 for details.
-
-	var (
-		sess     []session.Attachable
-		resolver remotes.Resolver
-	)
+	// Workaround: buildkit/containerd currently does not support pushing multi-image builds
+	//             with some registries, e.g. gcr.io. Until https://github.com/containerd/containerd/issues/5978
+	//             is resolved, we'll manually copy the image.
+	var resolver remotes.Resolver
 	if gplayerAuth := b.Config.WorkspaceLayerAuth; gplayerAuth != "" {
-		auth, err := newAuthProviderFromEnvvar(gplayerAuth)
+		_, err := newAuthProviderFromEnvvar(gplayerAuth)
 		if err != nil {
 			return err
 		}
-		sess = append(sess, auth)
 
 		authorizer, err := newDockerAuthorizerFromEnvvar(gplayerAuth)
 		if err != nil {
@@ -218,55 +215,212 @@ func (b *Builder) buildWorkspaceImage(ctx context.Context, cl *client.Client) (e
 	} else {
 		resolver = docker.NewResolver(docker.ResolverOptions{})
 	}
+	return copyImage(ctx, resolver, b.Config.BaseRef, b.Config.TargetRef)
+
+	// // Note: buildkit does not handle/export image config by default. That's why we need
+	// //       to download it ourselves and explicitely export it.
+	// //       See https://github.com/moby/buildkit/issues/2362 for details.
+	// var sess []session.Attachable
+	// if gplayerAuth := b.Config.WorkspaceLayerAuth; gplayerAuth != "" {
+	// 	auth, err := newAuthProviderFromEnvvar(gplayerAuth)
+	// 	if err != nil {
+	// 		return err
+	// 	}
+	// 	sess = append(sess, auth)
+
+	// 	authorizer, err := newDockerAuthorizerFromEnvvar(gplayerAuth)
+	// 	if err != nil {
+	// 		return err
+	// 	}
+	// 	resolver = docker.NewResolver(docker.ResolverOptions{
+	// 		Authorizer: authorizer,
+	// 	})
+	// } else {
+	// 	resolver = docker.NewResolver(docker.ResolverOptions{})
+	// }
+
+	// platform := specs.Platform{OS: "linux", Architecture: "amd64"}
+	// _, cfg, err := imageutil.Config(ctx, b.Config.BaseRef, resolver, contentutil.NewBuffer(), nil, &platform)
+	// if err != nil {
+	// 	return err
+	// }
+	// state, err := llb.Image(b.Config.BaseRef).WithImageConfig(cfg)
+	// if err != nil {
+	// 	return err
+	// }
+
+	// def, err := state.Marshal(ctx, llb.Platform(platform))
+	// if err != nil {
+	// 	return err
+	// }
+
+	// // TODO(cw):
+	// // buildkit does not support setting raw annotations yet (https://github.com/moby/buildkit/issues/1220).
+	// // Once it does, we should set org.opencontainers.image.base.name as defined in https://github.com/opencontainers/image-spec/blob/main/annotations.md
+
+	// solveOpt := client.SolveOpt{
+	// 	Exports: []client.ExportEntry{
+	// 		{
+	// 			Type: "image",
+	// 			Attrs: map[string]string{
+	// 				"name":                  b.Config.TargetRef,
+	// 				"push":                  "true",
+	// 				"containerimage.config": string(cfg),
+	// 			},
+	// 		},
+	// 	},
+	// 	Session:      sess,
+	// 	CacheImports: b.Config.LocalCacheImport(),
+	// }
+
+	// eg, ctx := errgroup.WithContext(ctx)
+	// ch := make(chan *client.SolveStatus)
+	// eg.Go(func() error {
+	// 	_, err := cl.Solve(ctx, def, solveOpt, ch)
+	// 	if err != nil {
+	// 		return xerrors.Errorf("cannot build Gitpod layer: %w", err)
+	// 	}
+	// 	return nil
+	// })
+	// eg.Go(func() error {
+	// 	var c console.Console
+	// 	return progressui.DisplaySolveStatus(ctx, "", c, os.Stdout, ch)
+	// })
+	// return eg.Wait()
+}
 
-	platform := specs.Platform{OS: "linux", Architecture: "amd64"}
-	_, cfg, err := imageutil.Config(ctx, b.Config.BaseRef, resolver, contentutil.NewBuffer(), nil, &platform)
+func copyImage(ctx context.Context, resolver remotes.Resolver, from, to string) error {
+	fromRef, fromDesc, err := resolver.Resolve(ctx, from)
 	if err != nil {
 		return err
 	}
-	state, err := llb.Image(b.Config.BaseRef).WithImageConfig(cfg)
+	fetcher, err := resolver.Fetcher(ctx, fromRef)
 	if err != nil {
 		return err
 	}
 
-	def, err := state.Marshal(ctx, llb.Platform(platform))
+	fetch := func(desc specs.Descriptor, out interface{}) error {
+		rc, err := fetcher.Fetch(ctx, fromDesc)
+		if err != nil {
+			return err
+		}
+		defer rc.Close()
+
+		return json.NewDecoder(rc).Decode(out)
+	}
+	if fromDesc.MediaType == specs.MediaTypeImageIndex {
+		var idx specs.Index
+		err := fetch(fromDesc, &idx)
+		if err != nil {
+			return err
+		}
+
+		var res *specs.Descriptor
+		for _, m := range idx.Manifests {
+			if m.Platform != nil && m.Platform.Architecture == "amd64" && m.Platform.OS == "linux" {
+				res = &m
+				break
+			}
+		}
+		if res == nil {
+			return fmt.Errorf("no manifest for amd64/linux found")
+		}
+		fromDesc = *res
+	}
+
+	var manifest specs.Manifest
+	err = fetch(fromDesc, &manifest)
+	if err != nil {
+		return err
+	}
+	manifestB, err := json.Marshal(manifest)
 	if err != nil {
 		return err
 	}
 
-	// TODO(cw):
-	// buildkit does not support setting raw annotations yet (https://github.com/moby/buildkit/issues/1220).
-	// Once it does, we should set org.opencontainers.image.base.name as defined in https://github.com/opencontainers/image-spec/blob/main/annotations.md
+	pusher, err := resolver.Pusher(ctx, to)
+	if err != nil {
+		return err
+	}
 
-	solveOpt := client.SolveOpt{
-		Exports: []client.ExportEntry{
-			{
-				Type: "image",
-				Attrs: map[string]string{
-					"name":                  b.Config.TargetRef,
-					"push":                  "true",
-					"containerimage.config": string(cfg),
-				},
-			},
-		},
-		Session:      sess,
-		CacheImports: b.Config.LocalCacheImport(),
+	// TODO(cw): optimize layer copy - copy only when pushing to a different registry
+	eg, egctx := errgroup.WithContext(ctx)
+	for _, layer := range manifest.Layers {
+		layer := layer
+		eg.Go(func() error { return copyLayer(egctx, fetcher, pusher, layer) })
+	}
+	eg.Go(func() error { return copyLayer(egctx, fetcher, pusher, manifest.Config) })
+	err = eg.Wait()
+	if err != nil {
+		return err
 	}
 
-	eg, ctx := errgroup.WithContext(ctx)
-	ch := make(chan *client.SolveStatus)
-	eg.Go(func() error {
-		_, err := cl.Solve(ctx, def, solveOpt, ch)
+	mfspec := specs.Descriptor{
+		MediaType: specs.MediaTypeImageManifest,
+		Digest:    digest.FromBytes(manifestB),
+		Size:      int64(len(manifestB)),
+		Platform:  &specs.Platform{OS: "linux", Architecture: "amd64"},
+	}
+	mfw, err := pusher.Push(ctx, mfspec)
+	if err != nil {
+		return err
+	}
+	defer mfw.Close()
+	mfwN, err := mfw.Write(manifestB)
+	if err != nil {
+		return err
+	}
+	if mfwN != len(manifestB) {
+		return fmt.Errorf("cannot write manifest: %w", io.ErrShortWrite)
+	}
+	err = mfw.Commit(ctx, mfspec.Size, mfspec.Digest)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func copyLayer(ctx context.Context, fetcher remotes.Fetcher, pusher remotes.Pusher, layer specs.Descriptor) (err error) {
+	log := log.WithField("blob", layer)
+	defer func() {
+		if errdefs.IsAlreadyExists(err) {
+			err = nil
+		}
 		if err != nil {
-			return xerrors.Errorf("cannot build Gitpod layer: %w", err)
+			log.WithError(err).Error("failed to copy blob")
+		} else {
+			log.Info("push complete")
 		}
-		return nil
-	})
-	eg.Go(func() error {
-		var c console.Console
-		return progressui.DisplaySolveStatus(ctx, "", c, os.Stdout, ch)
-	})
-	return eg.Wait()
+	}()
+	log.Info("pushing blob")
+
+	in, err := fetcher.Fetch(ctx, layer)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := pusher.Push(ctx, layer)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	n, err := io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+	if n != layer.Size {
+		return fmt.Errorf("copied less than %d bytes from layer %s (expected %d bytes)", n, layer.Digest.String(), layer.Size)
+	}
+
+	err = out.Commit(ctx, n, layer.Digest)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func waitForBuildContext(ctx context.Context) error {