cmd/coordinator: break up status into active vs pending builds

Currently all builds start and think they're running, but most are
just fighting over a mutex to grab a builder. That will be fixed, but
in the meantime it's nice to see what's actually working vs what's
waiting on e.g. arm5 hardware which won't be available for hours.

This is a baby step towards more monitoring. Currently this is just HTML
output, but the same data could be exported via JSON or something else later
for graphing.

Updates golang/go#19178 (add a buildlet scheduler)
Updates golang/go#15760 (monitor everything)

Change-Id: I36e16ea0919afe8023fe7fedd981f2e857f0d6df
Reviewed-on: https://go-review.googlesource.com/40397
Reviewed-by: Brad Fitzpatrick <[email protected]>

Author: bradfitz

cmd/coordinator/coordinator.go

@@ -39,6 +39,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"go4.org/syncutil"
@@ -1469,6 +1470,7 @@ func (st *buildStatus) build() error {
 	if err != nil {
 		return fmt.Errorf("failed to get a buildlet: %v", err)
 	}
+	atomic.StoreInt32(&st.hasBuildlet, 1)
 	defer bc.Close()
 	st.mu.Lock()
 	st.bc = bc
@@ -2874,6 +2876,8 @@ type buildStatus struct {
 	ctx             context.Context    // used to start the build
 	cancel          context.CancelFunc // used to cancel context; for use by setDone only
 
+	hasBuildlet int32 // atomic: non-zero if this build has a buildlet; for status.go.
+
 	mu              sync.Mutex       // guards following
 	failURL         string           // if non-empty, permanent URL of failure
 	bc              *buildlet.Client // nil initially, until pool returns one
@@ -3013,9 +3017,6 @@ func (st *buildStatus) htmlStatusLine(full bool) template.HTML {
 	defer st.mu.Unlock()
 
 	urlPrefix := "https://go-review.googlesource.com/#/q/"
-	if strings.Contains(st.name, "gccgo") {
-		urlPrefix = "https://code.google.com/p/gofrontend/source/detail?r="
-	}
 
 	var buf bytes.Buffer
 	fmt.Fprintf(&buf, "<a href='https://github.com/golang/go/wiki/DashboardBuilders'>%s</a> rev <a href='%s%s'>%s</a>",

cmd/coordinator/status.go

@@ -15,6 +15,7 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+	"sync/atomic"
 	"time"
 )
 
@@ -39,7 +40,12 @@ func handleStatus(w http.ResponseWriter, r *http.Request) {
 		NumGoroutine: runtime.NumGoroutine(),
 	}
 	for _, st := range status {
-		data.Active = append(data.Active, st)
+		if atomic.LoadInt32(&st.hasBuildlet) != 0 {
+			data.ActiveBuilds++
+			data.Active = append(data.Active, st)
+		} else {
+			data.Pending = append(data.Pending, st)
+		}
 	}
 	// TODO: make this prettier.
 	var buf bytes.Buffer
@@ -59,6 +65,7 @@ func handleStatus(w http.ResponseWriter, r *http.Request) {
 	data.RemoteBuildlets = template.HTML(remoteBuildletStatus())
 
 	sort.Sort(byAge(data.Active))
+	sort.Sort(byAge(data.Pending))
 	sort.Sort(sort.Reverse(byAge(data.Recent)))
 	if errTryDeps != nil {
 		data.TrybotsErr = errTryDeps.Error()
@@ -116,11 +123,13 @@ func diskFree() string {
 
 // statusData is the data that fills out statusTmpl.
 type statusData struct {
-	Total             int // number of total builds active
+	Total             int // number of total builds (including those waiting for a buildlet)
+	ActiveBuilds      int // number of running builds (subset of Total with a buildlet)
 	NumFD             int
 	NumGoroutine      int
 	Uptime            time.Duration
-	Active            []*buildStatus
+	Active            []*buildStatus // have a buildlet
+	Pending           []*buildStatus // waiting on a buildlet
 	Recent            []*buildStatus
 	TrybotsErr        string
 	Trybots           template.HTML
@@ -147,7 +156,7 @@ var statusTmpl = template.Must(template.New("status").Parse(`
 </header>
 
 <h2>Running</h2>
-<p>{{printf "%d" .Total}} total builds active. Uptime {{printf "%s" .Uptime}}. Version {{.Version}}.
+<p>{{printf "%d" .Total}} total builds; {{printf "%d" .ActiveBuilds}} active. Uptime {{printf "%s" .Uptime}}. Version {{.Version}}.
 
 <h2 id=trybots><a href='#trybots'>🔗</a> Active Trybot Runs</h2>
 {{- if .TrybotsErr}}
@@ -173,6 +182,13 @@ var statusTmpl = template.Must(template.New("status").Parse(`
 {{end}}
 </ul>
 
+<h2 id=pending><a href='#pending'>🔗</a> Pending builds</h2>
+<ul>
+{{range .Pending}}
+<li><pre>{{.HTMLStatusLine}}</pre></li>
+{{end}}
+</ul>
+
 <h2 id=completed><a href='#completed'>🔗</a> Recently completed</h2>
 <ul>
 {{range .Recent}}