Add support for building multi-arch images

Only install libipmctl for amd64 versions of the docker image.

Updated ipmctl to v02.00.00.3885 to fix intel/ipmctl#169

Updated base image to alpine:1.16 from alpine:1.15 to add zfs support on "arm" architecture.

Co-authored-by: David Porter <[email protected]>
Signed-of-by: Tyler Stapler <[email protected]>

Author: tstapler

.gitignore

@@ -1,6 +1,7 @@
 cadvisor
 /release
 .vscode
+_output/
 
 # Log files
 *.log

Makefile

@@ -17,13 +17,6 @@ GOLANGCI_VER := v1.45.2
 GO_TEST ?= $(GO) test $(or $(GO_FLAGS),-race)
 arch ?= $(shell go env GOARCH)
 
-ifeq ($(arch), amd64)
-  Dockerfile_tag := ''
-else
-  Dockerfile_tag := '.''$(arch)'
-endif
-
-
 all: presubmit build test
 
 test:
@@ -76,7 +69,7 @@ release:
 	@./build/release.sh
 
 docker-%:
-	@docker build -t cadvisor:$(shell git rev-parse --short HEAD) -f deploy/Dockerfile$(Dockerfile_tag) .
+	@docker build -t cadvisor:$(shell git rev-parse --short HEAD) -f deploy/Dockerfile .
 
 docker-build:
 	@docker run --rm -w /go/src/github.com/google/cadvisor -v ${PWD}:/go/src/github.com/google/cadvisor golang:1.18 make build
@@ -99,5 +92,6 @@ lint:
 
 clean:
 	@rm -f *.test cadvisor
+	@rm -rf _output/
 
 .PHONY: all build docker format release test test-integration lint presubmit tidy

build/build.sh

@@ -16,15 +16,17 @@
 
 set -e
 
+export GOOS=${GOOS:-$(go env GOOS)}
+export GOARCH=${GOARCH:-$(go env GOARCH)}
 GO_FLAGS=${GO_FLAGS:-"-tags netgo"}    # Extra go flags to use in the build.
 BUILD_USER=${BUILD_USER:-"${USER}@${HOSTNAME}"}
 BUILD_DATE=${BUILD_DATE:-$( date +%Y%m%d-%H:%M:%S )}
 VERBOSE=${VERBOSE:-}
-GOARCH=$1
+OUTPUT_NAME_WITH_ARCH=${OUTPUT_NAME_WITH_ARCH:-"false"}
 
 repo_path="github.com/google/cadvisor"
 
-version=$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' )
+version=${VERSION:-$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' )}
 revision=$( git rev-parse --short HEAD 2> /dev/null || echo 'unknown' )
 branch=$( git rev-parse --abbrev-ref HEAD 2> /dev/null || echo 'unknown' )
 go_version=$( go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/' )
@@ -50,15 +52,15 @@ if [ -n "$VERBOSE" ]; then
   echo "Building with -ldflags $ldflags"
 fi
 
+mkdir -p "$PWD/_output"
+output_file="$PWD/_output/cadvisor"
+if [ "${OUTPUT_NAME_WITH_ARCH}" = "true" ] ; then
+  output_file="${output_file}-${version}-${GOOS}-${GOARCH}"
+fi
+
 # Since github.com/google/cadvisor/cmd is a submodule, we must build from inside that directory
-output_file="$PWD/cadvisor"
 pushd cmd > /dev/null
-if [ -z "$GOARCH" ]
-then
   go build ${GO_FLAGS} -ldflags "${ldflags}" -o "${output_file}" "${repo_path}/cmd"
-else
-  env GOOS=linux GOARCH=$GOARCH go build ${GO_FLAGS} -ldflags "${ldflags}" -o "${output_file}" "${repo_path}/cmd"
-fi
 popd > /dev/null
 
 exit 0

build/check_container.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Description:
+# This script is meant to run a basic test against each of the CPU architectures
+# cadvisor should support.
+#
+# This script requires that you have run qemu-user-static so that your machine
+# can interpret ELF binaries for other architectures using QEMU:
+# https://github.com/multiarch/qemu-user-static#getting-started
+#
+# $ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+#
+# Usage:
+# ./check_container.sh gcr.io/tstapler-gke-dev/cadvisor:v0.44.1-test-4
+target_image=$1
+
+# Architectures officially supported by cadvisor
+arches=( "amd64" "arm" "arm64" )
+
+# Docker doesn't handle images with different architectures but the same tag.
+# Remove the container and the image use by it to avoid problems.
+cleanup() {
+    echo Cleaning up the container $1
+    docker stop $1
+    docker rmi $target_image
+    echo
+}
+
+for arch in "${arches[@]}"; do
+  echo Testing that we can run $1 on $arch and curl the /healthz endpoint
+  echo
+  container_id=$(docker run --platform "linux/$arch" -p 8080:8080 --rm --detach "$target_image")
+  docker_exit_code=$?
+  if [ $docker_exit_code -ne 0 ]; then
+    echo Failed to run container docker exited with $docker_exit_code
+    cleanup $container_id
+    exit $docker_exit_code
+  fi
+  sleep 10
+  echo
+  echo Testing the container with curl:
+  curl --show-error --retry 5 --fail -L 127.0.0.1:8080/healthz
+  echo
+  echo
+  curl_exit=$?
+  if [ $curl_exit -ne 0 ]; then
+    echo  Curling $target_image did not work
+    cleanup $container_id
+    exit $curl_exit
+  fi
+  echo Success!
+  echo
+  cleanup $container_id
+done

build/release.sh

@@ -16,12 +16,19 @@
 
 set -e
 
-VERSION=$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' )
-# Only allow releases of tagged versions.
-TAGGED='^v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.?[0-9]*)?$'
-if [[ ! "$VERSION" =~ $TAGGED ]]; then
-  echo "Error: Only tagged versions are allowed for releases" >&2
-  echo "Found: $VERSION" >&2
+if [ -z "$VERSION" ]; then
+  VERSION=$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' )
+  # Only allow releases of tagged versions.
+  TAGGED='^v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.?[0-9]*)?$'
+  if [[ ! "$VERSION" =~ $TAGGED ]]; then
+    echo "Error: Only tagged versions are allowed for releases" >&2
+    echo "Found: $VERSION" >&2
+    exit 1
+  fi
+fi
+
+read -p "Please confirm: $VERSION is the desired version (Type y/n to continue):" -n 1 -r
+if ! [[ $REPLY =~ ^[Yy]$ ]]; then
   exit 1
 fi
 
@@ -36,31 +43,51 @@ export BUILD_USER="$git_user"
 export BUILD_DATE=$( date +%Y%m%d ) # Release date is only to day-granularity
 export VERBOSE=true
 
-# Build the release binary with libpfm4 for docker container
-export GO_FLAGS="-tags=libpfm,netgo"
-build/build.sh
-
 # Build the docker image
 echo ">> building cadvisor docker image"
-gcr_tag="gcr.io/cadvisor/cadvisor:$VERSION"
-docker build -t $gcr_tag -f deploy/Dockerfile .
+image_name=${IMAGE_NAME:-"gcr.io/cadvisor/cadvisor"}
+final_image="$image_name:${VERSION}"
 
-# Build the release binary without libpfm4 to not require libpfm4 in runtime environment
-unset GO_FLAGS
-build/build.sh
+docker buildx rm cadvisor-builder || true
+docker buildx create --name cadvisor-builder
+docker buildx use cadvisor-builder
 
+# Build binaries
+
+# A mapping of the docker arch name to the qemu arch name
+declare -A arches=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64")
+
+for arch in "${arches[@]}"; do
+  if ! hash "qemu-${arch}-static"; then
+    echo Releasing multi arch containers requires qemu-user-static.
+    echo
+    echo Please install using apt-get install qemu-user-static or
+    echo a similar package for your OS.
+
+    exit 1
+  fi
+done
+
+for arch in "${!arches[@]}"; do
+  GOARCH="$arch" OUTPUT_NAME_WITH_ARCH="true" build/build.sh
+  arch_specific_image="${image_name}-${arch}:${VERSION}"
+  docker buildx build --platform "linux/${arch}" --build-arg VERSION="$VERSION" -f deploy/Dockerfile -t "$arch_specific_image"  --progress plain --push .
+  docker manifest create --amend "$final_image" "$arch_specific_image"
+  docker manifest annotate --os=linux --arch="$arch" "$final_image" "$arch_specific_image"
+done
+docker manifest push "$final_image"
+docker buildx rm cadvisor-builder
 echo
-echo "double-check the version below:"
-echo "VERSION=$VERSION"
-echo
-echo "To push docker image to gcr:"
-echo "docker push $gcr_tag"
+echo "Release info (copy to the release page)":
 echo
-echo "Release info (copy to the release page):"
+echo Multi Arch Container Image:
+echo $final_image
 echo
-echo "Docker Image: N/A"
-echo "gcr.io Image: $gcr_tag"
+echo Architecture Specific Container Images:
+for arch in "${!arches[@]}"; do
+  echo "${image_name}-${arch}:${VERSION}"
+done
 echo
-sha256sum --tag cadvisor
-
+echo Binaries:
+(cd _output && find . -name "cadvisor-${VERSION}*" -exec sha256sum --tag {} \;)
 exit 0

deploy/Dockerfile

@@ -1,8 +1,11 @@
-FROM alpine:3.15 AS build
+FROM mirror.gcr.io/library/golang:1.18-alpine3.16 AS build
 
-RUN apk --no-cache add libc6-compat device-mapper findutils zfs build-base linux-headers python3 bash git wget cmake pkgconfig ndctl-dev && \
+ARG VERSION
+
+# Install build depdencies for all supported arches
+RUN apk --no-cache add bash build-base cmake device-mapper findutils git \
+                       libc6-compat linux-headers ndctl-dev pkgconfig python3 wget zfs && \
     apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \
-    apk --no-cache add go==1.16.10-r0 --repository http://dl-3.alpinelinux.org/alpine/v3.14/community/ && \
     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
     rm -rf /var/cache/apk/*
 
@@ -15,35 +18,39 @@ RUN export DBG="-g -Wall" && \
   make -e -C libpfm-4.11.0 && \
   make install -C libpfm-4.11.0
 
-RUN git clone -b v02.00.00.3820 https://github.com/intel/ipmctl/ && \
+# ipmctl only supports Intel x86_64 processors.
+# https://github.com/intel/ipmctl/issues/163
+RUN if [ "$(uname --machine)" = "x86_64" ]; then \
+    git clone -b v02.00.00.3885 https://github.com/intel/ipmctl/ && \
     cd ipmctl && \
     mkdir output && \
     cd output && \
     cmake -DRELEASE=ON -DCMAKE_INSTALL_PREFIX=/ -DCMAKE_INSTALL_LIBDIR=/usr/local/lib .. && \
     make -j all && \
-    make install
+    make install; fi
 
 ADD . /go/src/github.com/google/cadvisor
 WORKDIR /go/src/github.com/google/cadvisor
 
-ENV GOROOT /usr/lib/go
-ENV GOPATH /go
-ENV GO_FLAGS="-tags=libpfm,netgo,libipmctl"
-
-RUN ./build/build.sh
+# libipmctl only works on x86_64 CPUs.
+RUN export GO_TAGS="-tags=libfpm,netgo"; \
+    if [ "$(uname --machine)" = "x86_64" ]; then \
+          export GO_TAGS="$GO_TAGS,libipmctl"; \
+    fi; \
+    GO_FLAGS="-tags=$GO_TAGS" ./build/build.sh
 
-FROM alpine:3.15
+FROM mirror.gcr.io/library/alpine:3.16
 MAINTAINER [email protected] [email protected] [email protected] [email protected] [email protected]
 
-RUN apk --no-cache add libc6-compat device-mapper findutils zfs ndctl && \
+RUN apk --no-cache add libc6-compat device-mapper findutils ndctl zfs && \
     apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \
     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
     rm -rf /var/cache/apk/*
 
-# Grab cadvisor,libpfm4 and libipmctl from "build" container.
+# Grab cadvisor,libpfm4 and libipmctl from "build" container if they exist (libipmctl only works on amd64/x86_64).
 COPY --from=build /usr/local/lib/libpfm.so* /usr/local/lib/
 COPY --from=build /usr/local/lib/libipmctl.so* /usr/local/lib/
-COPY --from=build /go/src/github.com/google/cadvisor/cadvisor /usr/bin/cadvisor
+COPY --from=build /go/src/github.com/google/cadvisor/_output/cadvisor /usr/bin/cadvisor
 
 EXPOSE 8080
 

deploy/Dockerfile.ppc64le

@@ -1,5 +1,7 @@
 FROM ppc64le/alpine:3.15
 MAINTAINER [email protected] [email protected]
+# Deprecated: the Dockerfile in this directory should support ppc64le
+# Simply build using: docker buildx build --platform linux/ppc64le -f Dockerfile .
 
 RUN apk --no-cache add libc6-compat device-mapper findutils zfs && \
     apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \

docs/development/releasing.md

@@ -51,32 +51,32 @@ Command: `make release`
 - Try to build it from the release branch, since we include that in the binary version
 - Verify the ldflags output, in particular check the Version, BuildUser, and GoVersion are expected
 
-Once the build is complete, check the VERSION and note the sha256 hash.
-
-## 4. Push the Docker images
-
-`make release` should output a command to push the image.  Alternatively, run:
+Once the build is complete, copy the output after `Release info...` and save it to use in step 5
 
+Example:
 ```
-$ PATCH_VERSION=v0.23.0
-$ docker push gcr.io/cadvisor/cadvisor:$PATCH_VERSION
+Multi Arch Container:
+gcr.io/cadvisor/cadvisor:v0.44.1-test-8
+
+Architecture Specific Containers:
+gcr.io/cadvisor/cadvisor-arm:v0.44.1-test-8
+gcr.io/cadvisor/cadvisor-arm64:v0.44.1-test-8
+gcr.io/cadvisor/cadvisor-amd64:v0.44.1-test-8
+
+Binaries:
+SHA256 (cadvisor-v0.44.1-test-8-linux-arm64) = e5e3f9e72208bc6a5ef8b837473f6c12877ace946e6f180bce8d81edadf66767
+SHA256 (cadvisor-v0.44.1-test-8-linux-arm) = 7d714e495a4f50d9cc374bd5e6b5c6922ffa40ff1cc7244f2308f7d351c4ccea
+SHA256 (cadvisor-v0.44.1-test-8-linux-amd64) = ea95c5a6db8eecb47379715c0ca260a8a8d1522971fd3736f80006c7f6cc9466
 ```
 
-## 5. Cut the release
+## 4. Cut the release
 
 Go to https://github.com/google/cadvisor/releases and click "Draft a new release"
 
 - "Tag version" and "Release title" should be preceded by 'v' and then the version. Select the tag pushed in step 2.b
 - Copy an old release as a template (e.g. github.com/google/cadvisor/releases/tag/v0.23.1)
 - Body should start with release notes (from CHANGELOG.md)
-- Next is the Docker image: `gcr.io/cadvisor/cadvisor:$VERSION`
-- Next are the binary hashes (from step 3)
-- Upload the binary build in step 3
+- Next are the docker images and binary hashes you copied (from step 3).
+- Upload the binaries build in step 3, they are located in the `_output` directory.
 - If this is a minor version release, mark the release as a "pre-release"
-- Click publish when done
-
-## 6. Finalize the release
-
-~~Once you are satisfied with the release quality (generally we wait until the next minor release), it is time to remove the "pre-release" tag~~
-
-cAdvisor is no longer pushed with the :latest tag.  This is to ensure tagged images are immutable.
+- Click publish when done