Skip to content

Commit 1f0362e

Browse files
mknyszekmknyszek
committed
dashboard: split misc-compile builders
This change splits up the misc-compile builders to only build for one platform each. This is the first step in a sequence of CLs to simplify misc-compile builders by eliminating the !SplitMakeRun paths, of which misc-compile builders are the only ones. The end result will be the support of misc-compile builders for subrepos. Splitting up misc-compile builders has the potential to increase VM demand, but each build also takes less time to run. The main way is could increase overall VM resource usage is the overhead of setting up and tearing down a VM. The other downside is more visual noise, though this is fairly minor. Note that this change also passes GOOS, GOARCH, and GOARM environment variables to buildall.sh. The goal is to eventually remove buildall.sh by just doing make.bash. (Right now buildall.sh will essentially ignore those variables and just set them itself.) For golang/go#58163. Change-Id: Ia48f6a16d080e9e078b2959dea3b68fe43def8ec Reviewed-on: https://go-review.googlesource.com/c/build/+/463777 Reviewed-by: Dmitri Shuralyov <[email protected]> Reviewed-by: Michael Pratt <[email protected]> Reviewed-by: Dmitri Shuralyov <[email protected]> Run-TryBot: Michael Knyszek <[email protected]> TryBot-Result: Gopher Robot <[email protected]>
1 parent 357881e commit 1f0362e

File tree

2 files changed

+209
-97
lines changed

2 files changed

+209
-97
lines changed

‎dashboard/builders.go

Lines changed: 80 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1519,70 +1519,89 @@ func init() {
15191519
})
15201520

15211521
// addMiscCompileGo1 adds a misc-compile TryBot that
1522-
// runs buildall.bash on the specified target(s), up to 3 max.
1523-
// The targets are matched against the "go tool dist list" name,
1524-
// but with hyphens instead of forward slashes ("linux-amd64", etc).
1522+
// runs buildall.bash on the specified target ("$goos-$goarch").
15251523
// If min is non-zero, it specifies the minimum Go 1.x version.
1526-
addMiscCompileGo1 := func(min int, suffix string, targets ...string) {
1527-
if len(targets) > 3 {
1528-
// This limit will do until we have better visibility
1529-
// into holistic TryBot completion times via metrics.
1530-
panic("at most 3 targets may be specified to avoid making TryBots slow; see issues 32632 and 17104")
1531-
}
1524+
addMiscCompileGo1 := func(min int, goos, goarch, extraSuffix string, extraEnv ...string) {
15321525
var v types.MajorMinor
15331526
var alsoNote string
15341527
if min != 0 {
15351528
v = types.MajorMinor{Major: 1, Minor: min}
15361529
alsoNote = fmt.Sprintf(" Applies to Go 1.%d and newer.", min)
15371530
}
1531+
// As an extra special case, add -arm5 to buildall.bash's filtering pattern for
1532+
// the linux-arm-arm5 case. buildall.bash treats this case specially and it explicitly
1533+
// unsets GOARM, so we can't just tell it what to do that way. For now, just do what
1534+
// it wants, but continue to pass GOARM for this case so that we can change buildall.bash
1535+
// safely.
1536+
//
1537+
// TODO(mknyszek): Fix buildall.bash to not explicitly unset GOARM. Consider rewriting it
1538+
// to better fit this new platform-per-builder model (which should simplify it a good bit).
1539+
filterSuffix := ""
1540+
if extraSuffix == "-arm5" {
1541+
filterSuffix = extraSuffix
1542+
}
1543+
platform := goos + "-" + goarch + extraSuffix
15381544
addBuilder(BuildConfig{
1539-
Name: "misc-compile" + suffix,
1540-
HostType: "host-linux-amd64-bullseye",
1541-
tryBot: defaultTrySet(),
1542-
env: []string{
1543-
"GO_DISABLE_OUTBOUND_NETWORK=1",
1544-
},
1545+
Name: "misc-compile-" + platform,
1546+
HostType: "host-linux-amd64-bullseye",
1547+
tryBot: defaultTrySet(),
1548+
env: append(extraEnv, "GO_DISABLE_OUTBOUND_NETWORK=1", "GOOS="+goos, "GOARCH="+goarch),
15451549
tryOnly: true,
15461550
MinimumGoVersion: v,
15471551
CompileOnly: true,
1548-
Notes: "Runs buildall.bash to cross-compile & vet std+cmd packages for " + strings.Join(targets, " & ") + ", but doesn't run any tests." + alsoNote,
1552+
Notes: "Runs buildall.bash to cross-compile & vet std+cmd packages (or runs 'go test -c' for subrepos) for " + platform + ", but doesn't run any tests." + alsoNote,
15491553
allScriptArgs: []string{
15501554
// Filtering pattern to buildall.bash:
1551-
"^(" + strings.Join(targets, "|") + ")$",
1555+
"^(" + goos + "-" + goarch + filterSuffix + ")$",
15521556
},
15531557
})
15541558
}
15551559
// addMiscCompile adds a misc-compile TryBot
15561560
// for all supported Go versions.
1557-
addMiscCompile := func(suffix string, targets ...string) { addMiscCompileGo1(0, suffix, targets...) }
1561+
addMiscCompile := func(goos, goarch string) { addMiscCompileGo1(0, goos, goarch, "") }
15581562

1559-
// Arrange so that no more than 3 ports are tested sequentially in each misc-compile
1560-
// TryBot to avoid any individual misc-compile TryBot from becoming a bottleneck for
1561-
// overall TryBot completion time (currently 10 minutes; see go.dev/issue/17104).
1563+
// To keep things simple, have each misc-compile builder handle exactly one platform.
15621564
//
1563-
// The TestTryBotsCompileAllPorts test is used to detect any gaps in TryBot coverage
1564-
// when new ports are added, and the misc-compile pairs below can be re-arranged.
1565+
// This is potentially wasteful as there could be much more VM creation overhead, but
1566+
// it shouldn't add any latency. It also adds some visual noise. The alternative was
1567+
// more complex support for subrepos; this keeps things simple by following the same
1568+
// general principle as all the other builders.
15651569
//
1566-
// (In the past, we used flexible regexp patterns that matched all architectures
1567-
// for a given GOOS value. However, over time as new architectures were added,
1568-
// some misc-compile TryBot could become much slower than others.)
1569-
//
1570-
// See go.dev/issue/32632.
1571-
addMiscCompile("-windows-arm", "windows-arm", "windows-arm64")
1572-
addMiscCompile("-darwin", "darwin-amd64", "darwin-arm64")
1573-
addMiscCompile("-mips", "linux-mips", "linux-mips64")
1574-
addMiscCompile("-mipsle", "linux-mipsle", "linux-mips64le")
1575-
addMiscCompile("-ppc", "linux-ppc64", "linux-ppc64le", "aix-ppc64")
1576-
addMiscCompile("-freebsd", "freebsd-386", "freebsd-arm", "freebsd-arm64")
1577-
addMiscCompile("-netbsd", "netbsd-386", "netbsd-amd64")
1578-
addMiscCompile("-netbsd-arm", "netbsd-arm", "netbsd-arm64")
1579-
addMiscCompile("-openbsd", "openbsd-386") // openbsd-mips64 go.dev/issue/58110
1580-
addMiscCompile("-openbsd-arm", "openbsd-arm", "openbsd-arm64")
1581-
addMiscCompile("-plan9", "plan9-386", "plan9-amd64", "plan9-arm")
1582-
addMiscCompile("-solaris", "solaris-amd64", "illumos-amd64")
1583-
addMiscCompile("-other-1", "dragonfly-amd64", "linux-loong64")
1584-
addMiscCompile("-other-2", "linux-riscv64", "linux-s390x", "linux-arm-arm5") // 'linux-arm-arm5' is linux/arm with GOARM=5.
1585-
addMiscCompileGo1(20, "-go1.20", "freebsd-riscv64")
1570+
// See https://go.dev/issue/58163 for more details.
1571+
addMiscCompile("windows", "arm")
1572+
addMiscCompile("windows", "arm64")
1573+
addMiscCompile("darwin", "amd64")
1574+
addMiscCompile("darwin", "arm64")
1575+
addMiscCompile("linux", "mips")
1576+
addMiscCompile("linux", "mips64")
1577+
addMiscCompile("linux", "mipsle")
1578+
addMiscCompile("linux", "mips64le")
1579+
addMiscCompile("linux", "ppc64")
1580+
addMiscCompile("linux", "ppc64le")
1581+
addMiscCompile("aix", "ppc64")
1582+
addMiscCompile("freebsd", "386")
1583+
addMiscCompile("freebsd", "arm")
1584+
addMiscCompile("freebsd", "arm64")
1585+
addMiscCompile("netbsd", "386")
1586+
addMiscCompile("netbsd", "amd64")
1587+
addMiscCompile("netbsd", "arm")
1588+
addMiscCompile("netbsd", "arm64")
1589+
addMiscCompile("openbsd", "386")
1590+
// openbsd-mips64 go.dev/issue/58110
1591+
addMiscCompile("openbsd", "arm")
1592+
addMiscCompile("openbsd", "arm64")
1593+
addMiscCompile("plan9", "386")
1594+
addMiscCompile("plan9", "amd64")
1595+
addMiscCompile("plan9", "arm")
1596+
addMiscCompile("solaris", "amd64")
1597+
addMiscCompile("illumos", "amd64")
1598+
addMiscCompile("dragonfly", "amd64")
1599+
addMiscCompile("linux", "loong64")
1600+
addMiscCompile("linux", "riscv64")
1601+
addMiscCompile("linux", "s390x")
1602+
addMiscCompile("linux", "arm")
1603+
addMiscCompileGo1(0, "linux", "arm", "-arm5", "GOARM=5")
1604+
addMiscCompileGo1(20, "freebsd", "riscv64", "-go1.20")
15861605

15871606
// TODO: Issue 25963, get the misc-compile trybots for Android/iOS.
15881607
// Then consider subrepos too, so "mobile" can at least be included
@@ -2996,22 +3015,35 @@ func addBuilder(c BuildConfig) {
29963015
// It adds a post-submit-only builder with KnownIssue, GoDeps set to the provided values,
29973016
// and runs on a limited set of branches to get test results without potential disruption
29983017
// for contributors. It can be modified as needed when onboarding a misc-compile builder.
2999-
func tryNewMiscCompile(suffix, rx string, knownIssue int, goDeps []string) {
3018+
func tryNewMiscCompile(goos, goarch, extraSuffix string, knownIssue int, goDeps []string, extraEnv ...string) {
30003019
if knownIssue == 0 {
30013020
panic("tryNewMiscCompile: knownIssue parameter must be non-zero")
30023021
}
3022+
// As an extra special case, add -arm5 to buildall.bash's filtering pattern for
3023+
// the linux-arm-arm5 case. buildall.bash treats this case specially and it explicitly
3024+
// unsets GOARM, so we can't just tell it what to do that way. For now, just do what
3025+
// it wants, but continue to pass GOARM for this case so that we can change buildall.bash
3026+
// safely.
3027+
//
3028+
// TODO(mknyszek): Fix buildall.bash to not explicitly unset GOARM. Consider rewriting it
3029+
// to better fit this new platform-per-builder model (which should simplify it a good bit).
3030+
filterSuffix := ""
3031+
if extraSuffix == "-arm5" {
3032+
filterSuffix = extraSuffix
3033+
}
3034+
platform := goos + "-" + goarch + extraSuffix
30033035
addBuilder(BuildConfig{
3004-
Name: "misc-compile" + suffix,
3036+
Name: "misc-compile" + platform,
30053037
HostType: "host-linux-amd64-bullseye",
30063038
buildsRepo: func(repo, branch, goBranch string) bool { return repo == "go" && branch == "master" },
30073039
KnownIssues: []int{knownIssue},
30083040
GoDeps: goDeps,
3009-
env: []string{"GO_DISABLE_OUTBOUND_NETWORK=1"},
3041+
env: append(extraEnv, "GOOS="+goos, "GOARCH="+goarch, "GO_DISABLE_OUTBOUND_NETWORK=1"),
30103042
CompileOnly: true,
3011-
Notes: fmt.Sprintf("Tries buildall.bash to cross-compile & vet std+cmd packages for "+rx+", but doesn't run any tests. See go.dev/issue/%d.", knownIssue),
3043+
Notes: fmt.Sprintf("Tries buildall.bash to cross-compile & vet std+cmd packages for "+platform+", but doesn't run any tests. See go.dev/issue/%d.", knownIssue),
30123044
allScriptArgs: []string{
30133045
// Filtering pattern to buildall.bash:
3014-
rx,
3046+
"^(" + goos + "-" + goarch + filterSuffix + ")$",
30153047
},
30163048
})
30173049
}

‎dashboard/builders_test.go

Lines changed: 129 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -105,21 +105,39 @@ func TestTrybots(t *testing.T) {
105105
"windows-386-2012",
106106
"windows-amd64-2016",
107107

108-
"misc-compile-darwin",
109-
"misc-compile-freebsd",
110108
"misc-compile-windows-arm",
111-
"misc-compile-mips",
112-
"misc-compile-mipsle",
113-
"misc-compile-netbsd",
109+
"misc-compile-windows-arm64",
110+
"misc-compile-darwin-amd64",
111+
"misc-compile-darwin-arm64",
112+
"misc-compile-linux-mips",
113+
"misc-compile-linux-mips64",
114+
"misc-compile-linux-mipsle",
115+
"misc-compile-linux-mips64le",
116+
"misc-compile-linux-ppc64",
117+
"misc-compile-linux-ppc64le",
118+
"misc-compile-aix-ppc64",
119+
"misc-compile-freebsd-386",
120+
"misc-compile-freebsd-arm",
121+
"misc-compile-freebsd-arm64",
122+
"misc-compile-netbsd-386",
123+
"misc-compile-netbsd-amd64",
114124
"misc-compile-netbsd-arm",
115-
"misc-compile-openbsd",
125+
"misc-compile-netbsd-arm64",
126+
"misc-compile-openbsd-386",
116127
"misc-compile-openbsd-arm",
117-
"misc-compile-plan9",
118-
"misc-compile-ppc",
119-
"misc-compile-solaris",
120-
"misc-compile-other-1",
121-
"misc-compile-other-2",
122-
"misc-compile-go1.20",
128+
"misc-compile-openbsd-arm64",
129+
"misc-compile-plan9-386",
130+
"misc-compile-plan9-amd64",
131+
"misc-compile-plan9-arm",
132+
"misc-compile-solaris-amd64",
133+
"misc-compile-illumos-amd64",
134+
"misc-compile-dragonfly-amd64",
135+
"misc-compile-linux-loong64",
136+
"misc-compile-linux-riscv64",
137+
"misc-compile-linux-s390x",
138+
"misc-compile-linux-arm",
139+
"misc-compile-linux-arm-arm5",
140+
"misc-compile-freebsd-riscv64-go1.20",
123141
},
124142
},
125143
{
@@ -138,21 +156,39 @@ func TestTrybots(t *testing.T) {
138156
"windows-386-2012",
139157
"windows-amd64-2016",
140158

141-
"misc-compile-darwin",
142-
"misc-compile-freebsd",
143159
"misc-compile-windows-arm",
144-
"misc-compile-mips",
145-
"misc-compile-mipsle",
146-
"misc-compile-netbsd",
160+
"misc-compile-windows-arm64",
161+
"misc-compile-darwin-amd64",
162+
"misc-compile-darwin-arm64",
163+
"misc-compile-linux-mips",
164+
"misc-compile-linux-mips64",
165+
"misc-compile-linux-mipsle",
166+
"misc-compile-linux-mips64le",
167+
"misc-compile-linux-ppc64",
168+
"misc-compile-linux-ppc64le",
169+
"misc-compile-aix-ppc64",
170+
"misc-compile-freebsd-386",
171+
"misc-compile-freebsd-arm",
172+
"misc-compile-freebsd-arm64",
173+
"misc-compile-netbsd-386",
174+
"misc-compile-netbsd-amd64",
147175
"misc-compile-netbsd-arm",
148-
"misc-compile-openbsd",
176+
"misc-compile-netbsd-arm64",
177+
"misc-compile-openbsd-386",
149178
"misc-compile-openbsd-arm",
150-
"misc-compile-plan9",
151-
"misc-compile-ppc",
152-
"misc-compile-solaris",
153-
"misc-compile-other-1",
154-
"misc-compile-other-2",
155-
"misc-compile-go1.20",
179+
"misc-compile-openbsd-arm64",
180+
"misc-compile-plan9-386",
181+
"misc-compile-plan9-amd64",
182+
"misc-compile-plan9-arm",
183+
"misc-compile-solaris-amd64",
184+
"misc-compile-illumos-amd64",
185+
"misc-compile-dragonfly-amd64",
186+
"misc-compile-linux-loong64",
187+
"misc-compile-linux-riscv64",
188+
"misc-compile-linux-s390x",
189+
"misc-compile-linux-arm",
190+
"misc-compile-linux-arm-arm5",
191+
"misc-compile-freebsd-riscv64-go1.20",
156192

157193
// Include longtest builders on Go repo release branches. See issue 37827.
158194
"linux-386-longtest",
@@ -179,21 +215,39 @@ func TestTrybots(t *testing.T) {
179215
"windows-386-2012",
180216
"windows-amd64-2016",
181217

182-
"misc-compile-darwin",
183-
"misc-compile-freebsd",
184218
"misc-compile-windows-arm",
185-
"misc-compile-mips",
186-
"misc-compile-mipsle",
187-
"misc-compile-netbsd",
219+
"misc-compile-windows-arm64",
220+
"misc-compile-darwin-amd64",
221+
"misc-compile-darwin-arm64",
222+
"misc-compile-linux-mips",
223+
"misc-compile-linux-mips64",
224+
"misc-compile-linux-mipsle",
225+
"misc-compile-linux-mips64le",
226+
"misc-compile-linux-ppc64",
227+
"misc-compile-linux-ppc64le",
228+
"misc-compile-aix-ppc64",
229+
"misc-compile-freebsd-386",
230+
"misc-compile-freebsd-arm",
231+
"misc-compile-freebsd-arm64",
232+
"misc-compile-netbsd-386",
233+
"misc-compile-netbsd-amd64",
188234
"misc-compile-netbsd-arm",
189-
"misc-compile-openbsd",
235+
"misc-compile-netbsd-arm64",
236+
"misc-compile-openbsd-386",
190237
"misc-compile-openbsd-arm",
191-
"misc-compile-plan9",
192-
"misc-compile-ppc",
193-
"misc-compile-solaris",
194-
"misc-compile-other-1",
195-
"misc-compile-other-2",
196-
"misc-compile-go1.20",
238+
"misc-compile-openbsd-arm64",
239+
"misc-compile-plan9-386",
240+
"misc-compile-plan9-amd64",
241+
"misc-compile-plan9-arm",
242+
"misc-compile-solaris-amd64",
243+
"misc-compile-illumos-amd64",
244+
"misc-compile-dragonfly-amd64",
245+
"misc-compile-linux-loong64",
246+
"misc-compile-linux-riscv64",
247+
"misc-compile-linux-s390x",
248+
"misc-compile-linux-arm",
249+
"misc-compile-linux-arm-arm5",
250+
"misc-compile-freebsd-riscv64-go1.20",
197251

198252
// Include longtest builders on Go repo release branches. See issue 37827.
199253
"linux-386-longtest",
@@ -218,20 +272,38 @@ func TestTrybots(t *testing.T) {
218272
"windows-386-2012-oldcc",
219273
"windows-amd64-2016-oldcc",
220274

221-
"misc-compile-darwin",
222-
"misc-compile-freebsd",
223275
"misc-compile-windows-arm",
224-
"misc-compile-mips",
225-
"misc-compile-mipsle",
226-
"misc-compile-netbsd",
276+
"misc-compile-windows-arm64",
277+
"misc-compile-darwin-amd64",
278+
"misc-compile-darwin-arm64",
279+
"misc-compile-linux-mips",
280+
"misc-compile-linux-mips64",
281+
"misc-compile-linux-mipsle",
282+
"misc-compile-linux-mips64le",
283+
"misc-compile-linux-ppc64",
284+
"misc-compile-linux-ppc64le",
285+
"misc-compile-aix-ppc64",
286+
"misc-compile-freebsd-386",
287+
"misc-compile-freebsd-arm",
288+
"misc-compile-freebsd-arm64",
289+
"misc-compile-netbsd-386",
290+
"misc-compile-netbsd-amd64",
227291
"misc-compile-netbsd-arm",
228-
"misc-compile-openbsd",
292+
"misc-compile-netbsd-arm64",
293+
"misc-compile-openbsd-386",
229294
"misc-compile-openbsd-arm",
230-
"misc-compile-plan9",
231-
"misc-compile-ppc",
232-
"misc-compile-solaris",
233-
"misc-compile-other-1",
234-
"misc-compile-other-2",
295+
"misc-compile-openbsd-arm64",
296+
"misc-compile-plan9-386",
297+
"misc-compile-plan9-amd64",
298+
"misc-compile-plan9-arm",
299+
"misc-compile-solaris-amd64",
300+
"misc-compile-illumos-amd64",
301+
"misc-compile-dragonfly-amd64",
302+
"misc-compile-linux-loong64",
303+
"misc-compile-linux-riscv64",
304+
"misc-compile-linux-s390x",
305+
"misc-compile-linux-arm",
306+
"misc-compile-linux-arm-arm5",
235307

236308
// Include longtest builders on Go repo release branches. See issue 37827.
237309
"linux-386-longtest",
@@ -906,8 +978,16 @@ func TestMiscCompileLinuxGOARM5(t *testing.T) {
906978
t.Fatalf("invalid misc-compile filtering pattern for builder %q: %q",
907979
b.Name, b.allScriptArgs[0])
908980
}
909-
if re.MatchString("linux-arm-arm5") {
910-
ok = true
981+
if !re.MatchString("linux-arm-arm5") {
982+
continue
983+
}
984+
for _, v := range b.env {
985+
if v == "GOARM=5" {
986+
ok = true
987+
break
988+
}
989+
}
990+
if ok {
911991
break
912992
}
913993
}

0 commit comments

Comments
 (0)