Skip to content

Commit 9a40b17

Browse files
authored
Merge pull request #1081 from justaugustus/gcbuilder
Fork test-infra gcbuilder and initial commit of krel gcbmgr
2 parents 07208a9 + 993db73 commit 9a40b17

File tree

21 files changed

+1109
-0
lines changed

21 files changed

+1109
-0
lines changed

BUILD.bazel

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,14 @@ filegroup(
2828
srcs = [
2929
":package-srcs",
3030
"//cmd/blocking-testgrid-tests:all-srcs",
31+
"//cmd/gcbuilder:all-srcs",
3132
"//cmd/krel:all-srcs",
3233
"//cmd/kubepkg:all-srcs",
3334
"//cmd/patch-announce:all-srcs",
3435
"//cmd/release-notes:all-srcs",
3536
"//lib:all-srcs",
3637
"//pkg/command:all-srcs",
38+
"//pkg/gcp/build:all-srcs",
3739
"//pkg/git:all-srcs",
3840
"//pkg/kubepkg:all-srcs",
3941
"//pkg/log:all-srcs",

cmd/gcbuilder/BUILD.bazel

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
2+
3+
filegroup(
4+
name = "package-srcs",
5+
srcs = glob(["**"]),
6+
tags = ["automanaged"],
7+
visibility = ["//visibility:private"],
8+
)
9+
10+
filegroup(
11+
name = "all-srcs",
12+
srcs = [
13+
":package-srcs",
14+
"//cmd/gcbuilder/cmd:all-srcs",
15+
],
16+
tags = ["automanaged"],
17+
visibility = ["//visibility:public"],
18+
)
19+
20+
go_library(
21+
name = "go_default_library",
22+
srcs = ["main.go"],
23+
importpath = "k8s.io/release/cmd/gcbuilder",
24+
visibility = ["//visibility:private"],
25+
deps = ["//cmd/gcbuilder/cmd:go_default_library"],
26+
)
27+
28+
go_binary(
29+
name = "gcbuilder",
30+
embed = [":go_default_library"],
31+
visibility = ["//visibility:public"],
32+
)

cmd/gcbuilder/Dockerfile

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Copyright 2019 The Kubernetes Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
FROM google/cloud-sdk:258.0.0-alpine
16+
COPY builder run.sh /
17+
CMD ["/run.sh"]

cmd/gcbuilder/Makefile

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2019 The Kubernetes Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
EXTRA_ARG =
16+
17+
push-prod:
18+
bazel run //images/builder -- $(EXTRA_ARG) --project=k8s-testimages --scratch-bucket=gs://k8s-testimages-scratch images/builder
19+
20+
push:
21+
bazel run //images/builder -- $(EXTRA_ARG) --allow-dirty images/builder
22+
23+
.PHONY: push push-prod help

cmd/gcbuilder/README.md

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# GCB Builder
2+
3+
This builder is sugar on top of `gcloud builds submit`. It offers the following features:
4+
5+
- Automatically injecting the standard commit-based tag (e.g. `20190403-dddd315ad-dirty`) as `_GIT_TAG`
6+
- Optionally blocking pushes of dirty builds
7+
- Uploading the working directory to GCS once and then reusing it for several builds
8+
- Building multiple variants simultaneously and optionally sending their output to files
9+
10+
A "variant" is a group of GCB substitutions grouped together to describe several ways to build a
11+
given image. They are optionally defined in `variants.yaml` in the same folder as the `Dockerfile`
12+
and `cloudbuild.yaml`. For example, a subset of the `kubekins-e2e` variants looks like this:
13+
14+
```yaml
15+
variants:
16+
'1.16':
17+
CONFIG: '1.16'
18+
GO_VERSION: 1.13.5
19+
K8S_RELEASE: stable-1.16
20+
BAZEL_VERSION: 0.23.2
21+
'1.15':
22+
CONFIG: '1.15'
23+
GO_VERSION: 1.12.12
24+
K8S_RELEASE: stable-1.15
25+
BAZEL_VERSION: 0.23.2
26+
```
27+
28+
By default, the image builder will build both the `1.15` and `1.16` groups simultaneously.
29+
If `--log-dir` is specified, it will write the build logs for each to `1.15.log` and `1.16.log`.
30+
31+
Alternatively, you can use `--variant` to build only one variant, e.g. `--variant 1.15`.
32+
33+
If no `variants.yaml` is specified, `cloudbuild.yaml` will be run once with no extra substitutions
34+
beyond `_GIT_TAG`.
35+
36+
## Usage
37+
38+
```shell
39+
bazel run //images/builder -- [options] path/to/build-directory/
40+
```
41+
42+
- `--allow-dirty`: If true, allow pushing dirty builds.
43+
- `--log-dir`: If provided, build logs will be sent to files in this directory instead of to stdout/stderr.
44+
- `--project`: If specified, use a non-default GCP project.
45+
- `--scratch-bucket`: If provided, the complete GCS path for Cloud Build to store scratch files (sources, logs). Necessary for upload reuse. If omitted, `gcloud` will create or reuse a bucket of its choosing.
46+
- `--variant`: If specified, build only the given variant. An error if no variants are defined.
47+
- `--env-passthrough`: Comma-separated list of specified environment variables to be passed to GCB as substitutions with an underscore (`_`) prefix. If the variable doesn't exist, the substitution will exist but be empty.
48+
- `--build-dir`: If provided, this directory will be uploaded as the source for the Google Cloud Build run.
49+
- `--gcb-config`: If provided, this will be used as the name of the Google Cloud Build config file.
50+
- `--no-source`: If true, no source will be uploaded with this build.
51+
52+
### A note about logging in Prow
53+
54+
Prow job logs can be viewed at a URI constructed as follows: `https://prow.k8s.io/view/gcs/kubernetes-jenkins/logs/<job-name>/<job-number>` e.g., https://prow.k8s.io/view/gcs/kubernetes-jenkins/logs/ci-kubernetes-prototype-build/1187171788975509509
55+
56+
When `--log-dir` is specified (which is the default case when running in Prow), the GCB build logs will be written to a set of log files, based on the variant(s).
57+
58+
For example:
59+
- No variant --> `build.log` (https://storage.googleapis.com/kubernetes-jenkins/logs/post-release-push-image-k8s-cloud-builder/1186437931728900096/artifacts/build.log)
60+
- Variant: `build-ci` --> `build-ci.log` (https://storage.googleapis.com/kubernetes-jenkins/logs/ci-kubernetes-prototype-build/1187156434249322500/artifacts/build-ci.log)
61+
62+
For single-variant jobs where the preference is to log directly to stdout (so that the log is instead visible in `https://prow.k8s.io/view/gcs/kubernetes-jenkins/logs/<job-name>/<job-number>`), `LOG_TO_STDOUT="y"` can be specified.

cmd/gcbuilder/ci-runner.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
# Copyright 2019 The Kubernetes Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -o errexit
17+
set -o nounset
18+
set -o pipefail
19+
20+
if [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]];then
21+
echo "Activating service account..."
22+
gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
23+
fi
24+
25+
echo "Executing builder, sending logs to ${ARTIFACTS}..."
26+
bazel run //images/builder -- --log-dir="${ARTIFACTS}" "$@"

cmd/gcbuilder/cloudbuild.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
steps:
2+
- name: golang:$_GO_VERSION
3+
args:
4+
- go
5+
- build
6+
- -o=./images/builder/builder
7+
- ./images/builder
8+
env:
9+
- CGO_ENABLED=0
10+
- GOOS=linux
11+
- GOARCH=amd64
12+
- GO111MODULE=on
13+
- GOPROXY=https://proxy.golang.org
14+
- GOSUMDB=sum.golang.org
15+
- name: gcr.io/cloud-builders/docker
16+
args:
17+
- build
18+
- --tag=gcr.io/$PROJECT_ID/image-builder:$_GIT_TAG
19+
- --tag=gcr.io/$PROJECT_ID/image-builder:latest
20+
- .
21+
dir: images/builder
22+
substitutions:
23+
_GIT_TAG: '12345'
24+
_GO_VERSION: 1.12.12
25+
images:
26+
- 'gcr.io/$PROJECT_ID/image-builder:$_GIT_TAG'
27+
- 'gcr.io/$PROJECT_ID/image-builder:latest'

cmd/gcbuilder/cmd/BUILD.bazel

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = ["root.go"],
6+
importpath = "k8s.io/release/cmd/gcbuilder/cmd",
7+
visibility = ["//visibility:public"],
8+
deps = [
9+
"//pkg/gcp/build:go_default_library",
10+
"//pkg/log:go_default_library",
11+
"@com_github_sirupsen_logrus//:go_default_library",
12+
"@com_github_spf13_cobra//:go_default_library",
13+
],
14+
)
15+
16+
go_test(
17+
name = "go_default_test",
18+
srcs = ["root_test.go"],
19+
embed = [":go_default_library"],
20+
deps = ["@com_github_stretchr_testify//require:go_default_library"],
21+
)
22+
23+
filegroup(
24+
name = "package-srcs",
25+
srcs = glob(["**"]),
26+
tags = ["automanaged"],
27+
visibility = ["//visibility:private"],
28+
)
29+
30+
filegroup(
31+
name = "all-srcs",
32+
srcs = [":package-srcs"],
33+
tags = ["automanaged"],
34+
visibility = ["//visibility:public"],
35+
)

cmd/gcbuilder/cmd/root.go

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cmd
18+
19+
import (
20+
"os"
21+
"path"
22+
"path/filepath"
23+
"strings"
24+
25+
"github.com/sirupsen/logrus"
26+
"github.com/spf13/cobra"
27+
28+
"k8s.io/release/pkg/gcp/build"
29+
"k8s.io/release/pkg/log"
30+
)
31+
32+
// rootCmd represents the base command when called without any subcommands
33+
var rootCmd = &cobra.Command{
34+
Use: "gcbuilder",
35+
Short: "gcbuilder",
36+
SilenceUsage: true,
37+
SilenceErrors: true,
38+
PersistentPreRunE: initLogging,
39+
RunE: func(cmd *cobra.Command, args []string) error {
40+
return run()
41+
},
42+
}
43+
44+
type rootOptions struct {
45+
logLevel string
46+
}
47+
48+
var (
49+
rootOpts = &rootOptions{}
50+
buildOpts = &build.Options{}
51+
)
52+
53+
// Execute adds all child commands to the root command and sets flags appropriately.
54+
// This is called by main.main(). It only needs to happen once to the rootCmd.
55+
func Execute() {
56+
if err := rootCmd.Execute(); err != nil {
57+
logrus.Fatal(err)
58+
}
59+
}
60+
61+
func init() {
62+
rootCmd.PersistentFlags().StringVar(&buildOpts.ConfigDir, "config-dir", "", "Configuration directory")
63+
rootCmd.PersistentFlags().StringVar(&buildOpts.BuildDir, "build-dir", "", "If provided, this directory will be uploaded as the source for the Google Cloud Build run.")
64+
rootCmd.PersistentFlags().StringVar(&buildOpts.CloudbuildFile, "gcb-config", "cloudbuild.yaml", "If provided, this will be used as the name of the Google Cloud Build config file.")
65+
rootCmd.PersistentFlags().StringVar(&buildOpts.LogDir, "log-dir", "", "If provided, build logs will be sent to files in this directory instead of to stdout/stderr.")
66+
rootCmd.PersistentFlags().StringVar(&buildOpts.ScratchBucket, "scratch-bucket", "", "The complete GCS path for Cloud Build to store scratch files (sources, logs).")
67+
rootCmd.PersistentFlags().StringVar(&buildOpts.Project, "project", "", "If specified, use a non-default GCP project.")
68+
rootCmd.PersistentFlags().BoolVar(&buildOpts.AllowDirty, "allow-dirty", false, "If true, allow pushing dirty builds.")
69+
rootCmd.PersistentFlags().BoolVar(&buildOpts.NoSource, "no-source", false, "If true, no source will be uploaded with this build.")
70+
rootCmd.PersistentFlags().StringVar(&buildOpts.Variant, "variant", "", "If specified, build only the given variant. An error if no variants are defined.")
71+
rootCmd.PersistentFlags().StringVar(&buildOpts.EnvPassthrough, "env-passthrough", "", "Comma-separated list of specified environment variables to be passed to GCB as substitutions with an _ prefix. If the variable doesn't exist, the substitution will exist but be empty.")
72+
rootCmd.PersistentFlags().StringVar(&rootOpts.logLevel, "log-level", "info", "the logging verbosity, either 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug' or 'trace'")
73+
74+
buildOpts.ConfigDir = strings.TrimSuffix(buildOpts.ConfigDir, "/")
75+
}
76+
77+
// TODO: Clean up error handling
78+
func run() error {
79+
if buildOpts.ConfigDir == "" {
80+
logrus.Info("expected a config directory to be provided")
81+
// TODO: Should return error
82+
//nolint:gocritic
83+
return nil // errors.New("expected a config directory to be provided")
84+
}
85+
86+
if bazelWorkspace := os.Getenv("BUILD_WORKSPACE_DIRECTORY"); bazelWorkspace != "" {
87+
if err := os.Chdir(bazelWorkspace); err != nil {
88+
logrus.Fatalf("Failed to chdir to bazel workspace (%s): %v", bazelWorkspace, err)
89+
}
90+
}
91+
92+
if buildOpts.BuildDir == "" {
93+
buildOpts.BuildDir = buildOpts.ConfigDir
94+
}
95+
96+
logrus.Infof("Build directory: %s\n", buildOpts.BuildDir)
97+
98+
// Canonicalize the config directory to be an absolute path.
99+
// As we're about to cd into the build directory, we need a consistent way to reference the config files
100+
// when the config directory is not the same as the build directory.
101+
absConfigDir, absErr := filepath.Abs(buildOpts.ConfigDir)
102+
if absErr != nil {
103+
logrus.Fatalf("Could not resolve absolute path for config directory: %v", absErr)
104+
}
105+
106+
buildOpts.ConfigDir = absConfigDir
107+
buildOpts.CloudbuildFile = path.Join(buildOpts.ConfigDir, buildOpts.CloudbuildFile)
108+
109+
configDirErr := buildOpts.ValidateConfigDir()
110+
if configDirErr != nil {
111+
logrus.Fatalf("Could not validate config directory: %v", configDirErr)
112+
}
113+
114+
logrus.Infof("Config directory: %s\n", buildOpts.ConfigDir)
115+
116+
logrus.Infof("cd-ing to build directory: %s\n", buildOpts.BuildDir)
117+
if err := os.Chdir(buildOpts.BuildDir); err != nil {
118+
logrus.Fatalf("Failed to chdir to build directory (%s): %v", buildOpts.BuildDir, err)
119+
}
120+
121+
buildErrors := build.RunBuildJobs(buildOpts)
122+
if len(buildErrors) != 0 {
123+
logrus.Fatalf("Failed to run some build jobs: %v", buildErrors)
124+
}
125+
logrus.Info("Finished.")
126+
127+
return nil
128+
}
129+
130+
func initLogging(*cobra.Command, []string) error {
131+
return log.SetupGlobalLogger(rootOpts.logLevel)
132+
}

0 commit comments

Comments
 (0)