cmd/scaleway: run instance checker on Kubernetes regularly

Fixes golang/go#21238

Change-Id: Ie3b3f240acb8f7578143f2594f4aaac94baeb0c3
Reviewed-on: https://go-review.googlesource.com/c/build/+/170441
Reviewed-by: Dmitri Shuralyov <[email protected]>

Author: bradfitz

cmd/scaleway/Dockerfile

@@ -1,9 +1,32 @@
 # Copyright 2017 The Go Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
-FROM scratch
+
+FROM golang:1.12-stretch AS build
 LABEL maintainer "[email protected]"
 
-COPY ca-certificates.crt /etc/ssl/certs/
-COPY scaleway /
-ENTRYPOINT ["/scaleway"]
+# BEGIN deps (run `make update-deps` to update)
+
+# Repo go4.org at ce4c26f (2019-02-17)
+ENV REV=ce4c26f7be8eb27dc77f996b08d286dd80bc4a01
+RUN go get -d go4.org/types &&\
+    (cd /go/src/go4.org && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
+
+# Optimization to speed up iterative development, not necessary for correctness:
+RUN go install go4.org/types
+# END deps.
+
+COPY . /go/src/golang.org/x/build/
+
+RUN go install -ldflags "-X 'main.Version=$version'" golang.org/x/build/cmd/scaleway
+
+
+FROM debian:stretch
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    netbase \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=build /go/bin/scaleway /scaleway

cmd/scaleway/Dockerfile.0

@@ -3,37 +3,24 @@
 # license that can be found in the LICENSE file.
 
 VERSION=latest
-DOCKER_IMAGE_build0=build0/scaleway:latest
-DOCKER_CTR_build0=scaleway-build0
-
-build0: *.go Dockerfile.0
-	docker build --force-rm -f Dockerfile.0 --tag=$(DOCKER_IMAGE_build0) ../..
-
-scaleway: build0
-	docker create --name $(DOCKER_CTR_build0) $(DOCKER_IMAGE_build0)
-	docker cp $(DOCKER_CTR_build0):/go/bin/$@ $@
-	docker rm $(DOCKER_CTR_build0)
-
-ca-certificates.crt:
-	docker create --name $(DOCKER_CTR_build0) $(DOCKER_IMAGE_build0)
-	docker cp $(DOCKER_CTR_build0):/etc/ssl/certs/$@ $@
-	docker rm $(DOCKER_CTR_build0)
 
 update-deps:
 	go install golang.org/x/build/cmd/gitlock
-	gitlock --update=Dockerfile.0 golang.org/x/build/cmd/scaleway
+	gitlock --update=Dockerfile golang.org/x/build/cmd/scaleway
 
-docker-prod: Dockerfile scaleway ca-certificates.crt
-	docker build --force-rm --tag=gcr.io/symbolic-datum-552/scaleway:$(VERSION) .
-docker-dev: Dockerfile scaleway ca-certificates.crt
-	docker build --force-rm --tag=gcr.io/go-dashboard-dev/scaleway:latest .
+docker: Dockerfile *.go
+	xb docker build -f Dockerfile --force-rm --tag=golang/scaleway:$(VERSION) ../..
 
-push-prod: docker-prod
+push-prod: docker
+	docker tag golang/scaleway:$(VERSION) gcr.io/symbolic-datum-552/scaleway:$(VERSION)
 	docker push gcr.io/symbolic-datum-552/scaleway:$(VERSION)
-push-dev: docker-dev
-	docker push gcr.io/go-dashboard-dev/scaleway:latest
 
-.PHONY: clean
-clean:
-	$(RM) scaleway
-	$(RM) ca-certificates.crt
+stop-prod:
+	xb --prod kubectl delete -f deployment-prod.yaml
+
+start-prod:
+	xb --prod kubectl create -f deployment-prod.yaml
+
+restart-prod:
+	xb --prod kubectl delete -f deployment-prod.yaml || true
+	xb --prod kubectl create -f deployment-prod.yaml

cmd/scaleway/Makefile

@@ -5,3 +5,7 @@
 # golang.org/x/build/cmd/scaleway
 
 The scaleway command creates ARM servers on Scaleway.com.
+
+The Makefile, Dockerfiles, and kubernetes deployment file
+are for running the command in daemon mode for server repair.
+This will check on Scaleway and restart down or wedged instances.

cmd/scaleway/README

@@ -12,6 +12,7 @@ spec:
         container.seccomp.security.alpha.kubernetes.io/scaleway: docker/default
         container.apparmor.security.beta.kubernetes.io/scaleway: runtime/default
     spec:
+      restartPolicy: Always
       volumes:
         - name: secrets-volume
           secret:
@@ -25,7 +26,7 @@ spec:
         - name: scaleway
           image: gcr.io/symbolic-datum-552/scaleway:latest
           imagePullPolicy: Always
-          command: ["/scaleway", "-daemon"]
+          command: ["/scaleway", "-daemon", "-token-dir=/keys"]
           volumeMounts:
             - name: secrets-volume
               mountPath: /keys

cmd/scaleway/README.md

@@ -26,7 +26,8 @@ import (
 )
 
 var (
-	token       = flag.String("token", "", "API token. If empty, the file is read from $HOME/keys/go-scaleway.token. Googlers on the Go team can get the value from http://go/golang-scaleway-token")
+	tokenDir    = flag.String("token-dir", filepath.Join(os.Getenv("HOME"), "keys"), "directory to read gobuilder-staging.key, gobuilder-master.key and go-scaleway.token from.")
+	token       = flag.String("token", "", "API token. If empty, the file is read from $(token-dir)/go-scaleway.token. Googlers on the Go team can get the value from http://go/golang-scaleway-token")
 	org         = flag.String("org", "1f34701d-668b-441b-bf08-0b13544e99de", "Organization ID (default is [email protected]'s account)")
 	image       = flag.String("image", "e488d5e3-d278-47a7-8f7d-1154e1f61dc9", "Disk image ID; default is the snapshot we made last")
 	num         = flag.Int("n", 0, "Number of servers to create; if zero, defaults to a value as a function of --staging")
@@ -62,7 +63,7 @@ func main() {
 		}
 	}
 	if *token == "" {
-		file := filepath.Join(os.Getenv("HOME"), "keys/go-scaleway.token")
+		file := filepath.Join(*tokenDir, "go-scaleway.token")
 		slurp, err := ioutil.ReadFile(file)
 		if err != nil {
 			if os.IsNotExist(err) {
@@ -74,6 +75,9 @@ func main() {
 	}
 
 	// Loop over checkServers() in daemon mode.
+	if *daemonMode {
+		log.Printf("scaleway instance checker daemon running.")
+	}
 	for {
 		checkServers()
 		if !*daemonMode {
@@ -84,6 +88,9 @@ func main() {
 }
 
 func checkServers() {
+	timer := time.AfterFunc(5*time.Minute, func() { panic("Timeout running checkServers.") })
+	defer timer.Stop()
+
 	cl := &Client{Token: *token}
 	serverList, err := cl.Servers()
 	if err != nil {
@@ -307,7 +314,7 @@ func (ip *IP) String() string {
 // defaultBuilderTags returns the default value of the "tags" flag.
 // It returns a comma-separated list of builder tags (each of the form buildkey_$(BUILDER)_$(SECRETHEX)).
 func defaultBuilderTags(baseKeyFile string) string {
-	keyFile := filepath.Join(os.Getenv("HOME"), "keys", baseKeyFile)
+	keyFile := filepath.Join(*tokenDir, baseKeyFile)
 	slurp, err := ioutil.ReadFile(keyFile)
 	if err != nil {
 		log.Fatal(err)

cmd/scaleway/deployment-prod.yaml

@@ -147,7 +147,7 @@ func runDocker() {
 			case "golang/buildlet-stage0":
 				log.Printf("building dependent layer %q", layer)
 				buildStage0Container()
-			case "debian:jessie", "debian:stretch", "debian:buster":
+			case "debian:jessie", "debian:stretch", "debian:buster", "golang:1.12-stretch":
 				// TODO: validate version? probably doesn't matter, as they're
 				// pretty frozen and just get security/bug updates, and most of
 				// our Dockerfiles start with apt-get update && upgrade steps