images: Migrate go-runner image building to k/release

BUILD.bazel

@@ -34,6 +34,7 @@ filegroup(
         "//cmd/kubepkg:all-srcs",
         "//cmd/release-notes:all-srcs",
         "//cmd/schedule-builder:all-srcs",
+        "//images/build/go-runner:all-srcs",
         "//lib:all-srcs",
         "//packages/deb:all-srcs",
         "//pkg/announce:all-srcs",

dependencies.yaml

@@ -98,3 +98,11 @@ dependencies:
     refPaths:
     - path: images/build/debian-hyperkube-base/Makefile
       match: BASEIMAGE\?\=\$\(BASE_REGISTRY\)\/debian-iptables-\$\(ARCH\):v\d+\.\d+\.\d+
+
+  - name: "k8s.gcr.io/go-runner"
+    version: buster-v1.0.0
+    refPaths:
+    - path: images/build/go-runner/Makefile
+      match: IMAGE_VERSION\ \?=\ [a-zA-Z]+\-v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)
+    - path: images/build/go-runner/variants.yaml
+      match: v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)

images/build/go-runner/BUILD.bazel

@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["go-runner.go"],
+    importpath = "k8s.io/release/images/build/go-runner",
+    visibility = ["//visibility:private"],
+    deps = ["@com_github_pkg_errors//:go_default_library"],
+)
+
+go_binary(
+    name = "go-runner",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

images/build/go-runner/Dockerfile

@@ -0,0 +1,48 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the manager binary
+ARG GO_VERSION
+ARG DISTROLESS_IMAGE
+FROM golang:${GO_VERSION} as builder
+WORKDIR /workspace
+
+# Run this with docker build --build_arg goproxy=$(go env GOPROXY) to override the goproxy
+ARG goproxy=https://proxy.golang.org
+# Run this with docker build --build_arg package=./controlplane/kubeadm or --build_arg package=./bootstrap/kubeadm
+ENV GOPROXY=$goproxy
+
+# Copy the sources
+COPY ./go-runner.go ./
+COPY ./go.* ./
+
+# Cache the go build
+RUN go build .
+
+# Build
+ARG package=.
+ARG ARCH
+
+# Do not force rebuild of up-to-date packages (do not use -a)
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
+    go build -ldflags '-s -w -buildid= -extldflags "-static"' \
+    -o go-runner ${package}
+
+# Production image
+FROM gcr.io/distroless/${DISTROLESS_IMAGE}:latest
+LABEL maintainers="Kubernetes Authors"
+LABEL description="go based runner for distroless scenarios"
+WORKDIR /
+COPY --from=builder /workspace/go-runner .
+ENTRYPOINT ["/go-runner"]

images/build/go-runner/Makefile

@@ -0,0 +1,85 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# set default shell
+SHELL=/bin/bash -o pipefail
+
+REGISTRY ?= gcr.io/k8s-staging-build-image
+IMGNAME = go-runner
+IMAGE = $(REGISTRY)/$(IMGNAME)
+
+TAG ?= $(shell git describe --tags --always --dirty)
+IMAGE_VERSION ?= buster-v1.0.0
+CONFIG ?= buster
+
+# Build args
+GO_VERSION ?= 1.13.15
+DISTROLESS_IMAGE ?= static-debian10
+
+PLATFORMS = linux/amd64 linux/arm64 linux/arm linux/ppc64le linux/s390x
+
+HOST_GOOS ?= $(shell go env GOOS)
+HOST_GOARCH ?= $(shell go env GOARCH)
+GO_BUILD ?= go build
+
+.PHONY: all build clean
+
+.PHONY: all
+all: build
+
+.PHONY: build
+build:
+	$(GO_BUILD)
+
+.PHONY: clean
+clean:
+	rm go-runner
+
+.PHONY: container
+container: init-docker-buildx
+	# https://github.com/docker/buildx/issues/59
+	$(foreach PLATFORM,$(PLATFORMS), \
+		DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build \
+		--load \
+		--progress plain \
+		--platform $(PLATFORM) \
+		--tag $(IMAGE)-$(PLATFORM):$(IMAGE_VERSION) \
+		--tag $(IMAGE)-$(PLATFORM):$(TAG)-$(CONFIG) \
+		--tag $(IMAGE)-$(PLATFORM):latest-$(CONFIG) \
+		--build-arg=GO_VERSION=$(GO_VERSION) \
+		--build-arg=DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) .;)
+
+.PHONY: push
+push: container
+	$(foreach PLATFORM,$(PLATFORMS), \
+		docker push $(IMAGE)-$(PLATFORM):$(IMAGE_VERSION);)
+	$(foreach PLATFORM,$(PLATFORMS), \
+		docker push $(IMAGE)-$(PLATFORM):$(TAG)-$(CONFIG);)
+	$(foreach PLATFORM,$(PLATFORMS), \
+		docker push $(IMAGE)-$(PLATFORM):latest-$(CONFIG);)
+
+.PHONY: manifest
+manifest: push
+	docker manifest create --amend $(IMAGE):$(IMAGE_VERSION) $(shell echo $(PLATFORMS) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(IMAGE_VERSION)~g")
+	@for arch in $(PLATFORMS); do docker manifest annotate --arch "$${arch##*/}" ${IMAGE}:${IMAGE_VERSION} ${IMAGE}-$${arch}:${IMAGE_VERSION}; done
+	docker manifest push --purge $(IMAGE):$(IMAGE_VERSION)
+
+.PHONY: init-docker-buildx
+init-docker-buildx:
+ifneq ($(shell docker buildx 2>&1 >/dev/null; echo $?),)
+	$(error "buildx not vailable. Docker 19.03 or higher is required")
+endif
+	docker run --rm --privileged linuxkit/binfmt:4ea3b9b0938cbd19834c096aa31ff475cc75d281
+	docker buildx create --name multiarch-go-runner --use || true
+	docker buildx inspect --bootstrap

images/build/go-runner/OWNERS

@@ -0,0 +1,6 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - build-image-approvers
+reviewers:
+  - build-image-reviewers

images/build/go-runner/README.md

@@ -0,0 +1,30 @@
+# Kubernetes go-runner image
+
+The Kubernetes go-runner image wraps the gcr.io/distroless/static image and provides a go based
+binary that can run commands and wrap stdout/stderr etc. 
+
+Why do we need this? Some of our images like kube-apiserver currently use bash for collecting
+logs, so we are not able to switch to distroless images directly for these images. The klog's
+`--log-file` was supposed to fix this problem, but we ran into trouble in scalability CI jobs
+around log rotation and picked this option instead. we essentially publish a multi-arch 
+manifest with support for various platforms. This can be used as a base for other kubernetes
+components.
+
+For example instead of running kube-apiserver like this:
+```bash
+"/bin/sh",
+  "-c",
+  "exec /usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1"
+```  
+
+we would use go-runner like so:
+```bash
+"/go-runner", "--log-file=/var/log/kube-apiserver.log", "--also-stdout=false", "--redirect-stderr=true",
+  "/usr/local/bin/kube-apiserver",
+  "--allow-privileged={{pillar['allow_privileged']}}",
+  {{params}}
+```
+
+The go-runner would then ensure that we run the `/usr/local/bin/kube-apiserver` with the 
+specified parameters and redirect stdout ONLY to the log file specified and ensure anything 
+logged to stderr also ends up in the log file.

images/build/go-runner/cloudbuild.yaml

@@ -0,0 +1,51 @@
+# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md for more details on image pushing process
+
+# this must be specified in seconds. If omitted, defaults to 600s (10 mins)
+timeout: 1200s
+# this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
+# or any new substitutions added in the future.
+options:
+  substitution_option: ALLOW_LOOSE
+  machineType: 'N1_HIGHCPU_8'
+steps:
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200422-b25d964'
+    entrypoint: 'bash'
+    dir: ./images/build/go-runner
+    env:
+    - DOCKER_CLI_EXPERIMENTAL=enabled
+    - REGISTRY=gcr.io/$PROJECT_ID
+    - HOME=/root
+    - TAG=$_GIT_TAG
+    - PULL_BASE_REF=$_PULL_BASE_REF
+    - IMAGE_VERSION=$_IMAGE_VERSION
+    - CONFIG=$_CONFIG
+    - GO_VERSION=$_GO_VERSION
+    - DISTROLESS_IMAGE=$_DISTROLESS_IMAGE
+    args:
+    - '-c'
+    - |
+      gcloud auth configure-docker \
+      && make manifest
+
+substitutions:
+  # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
+  # can be used as a substitution
+  _GIT_TAG: '12345'
+  _PULL_BASE_REF: 'dev'
+  _IMAGE_VERSION: 'codename-v0.0.0'
+  _CONFIG: 'codename'
+  _GO_VERSION: '0.0.0'
+  _DISTROLESS_IMAGE: 'static-debian00'
+
+tags:
+- ${_GIT_TAG}
+- ${_PULL_BASE_REF}
+- ${_IMAGE_VERSION}
+- ${_CONFIG}
+- ${_GO_VERSION}
+- ${_DISTROLESS_IMAGE}
+
+images:
+  - 'gcr.io/$PROJECT_ID/go-runner-linux/amd64:$_IMAGE_VERSION'
+  - 'gcr.io/$PROJECT_ID/go-runner-linux/amd64:$_GIT_TAG-$_CONFIG'
+  - 'gcr.io/$PROJECT_ID/go-runner-linux/amd64:latest-$_CONFIG'

images/build/go-runner/go-runner.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	logFilePath    = flag.String("log-file", "", "If non-empty, save stdout to this file")
+	alsoToStdOut   = flag.Bool("also-stdout", false, "useful with log-file, log to standard output as well as the log file")
+	redirectStderr = flag.Bool("redirect-stderr", true, "treat stderr same as stdout")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := configureAndRun(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func configureAndRun() error {
+	var (
+		outputStream io.Writer = os.Stdout
+		errStream    io.Writer = os.Stderr
+	)
+
+	args := flag.Args()
+	if len(args) == 0 {
+		return errors.Errorf("not enough arguments to run")
+	}
+
+	if logFilePath != nil && *logFilePath != "" {
+		logFile, err := os.OpenFile(*logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			return errors.Wrapf(err, "failed to create log file %v", *logFilePath)
+		}
+		if *alsoToStdOut {
+			outputStream = io.MultiWriter(os.Stdout, logFile)
+		} else {
+			outputStream = logFile
+		}
+	}
+
+	if *redirectStderr {
+		errStream = outputStream
+	}
+
+	exe := args[0]
+	var exeArgs []string
+	if len(args) > 1 {
+		exeArgs = args[1:]
+	}
+	cmd := exec.Command(exe, exeArgs...)
+	cmd.Stdout = outputStream
+	cmd.Stderr = errStream
+
+	log.Printf("Running command:\n%v", cmdInfo(cmd))
+	err := cmd.Start()
+	if err != nil {
+		return errors.Wrap(err, "starting command")
+	}
+
+	// Handle signals and shutdown process gracefully.
+	go setupSigHandler(cmd.Process)
+	return errors.Wrap(cmd.Wait(), "running command")
+}
+
+// cmdInfo generates a useful look at what the command is for printing/debug.
+func cmdInfo(cmd *exec.Cmd) string {
+	return fmt.Sprintf(
+		`Command env: (log-file=%v, also-stdout=%v, redirect-stderr=%v)
+Run from directory: %v
+Executable path: %v
+Args (comma-delimited): %v`, *logFilePath, *alsoToStdOut, *redirectStderr,
+		cmd.Dir, cmd.Path, strings.Join(cmd.Args, ","),
+	)
+}
+
+// setupSigHandler will forward any termination signals to the process
+func setupSigHandler(process *os.Process) {
+	// terminationSignals are signals that cause the program to exit in the
+	// supported platforms (linux, darwin, windows).
+	terminationSignals := []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT}
+
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, terminationSignals...)
+
+	// Block until a signal is received.
+	log.Println("Now listening for interrupts")
+	s := <-c
+	log.Printf("Got signal: %v. Sending down to process (PID: %v)", s, process.Pid)
+	if err := process.Signal(s); err != nil {
+		log.Fatalf("Failed to signal process: %v", err)
+	}
+	log.Printf("Signalled process %v successfully.", process.Pid)
+}