runtime: bound scanobject to ~100 µs

DO NOT SUBMIT. Benchmarks (preferably including mutator latency).

Currently the time spent in scanobject is proportional to the size of
the object being scanned. Since this time is non-preemptible, large
objects can cause significant mutator delays, particularly if a
mutator assist picks up a large object.

Fix this by splitting large objects into 128 KB "oblets" and scanning
at most one oblet at a time. Since we can scan 1–2 MB/ms, this equates
to bounding scanobject at roughly 100 µs. This also improves GC
parallelism if the heap consists primarily of a small number of very
large objects.

Fixes #10345.

Change-Id: I2a0a179d1d6bf7875dd054b7693dd12d2a340132

Author: aclements

src/runtime/mgcmark.go

@@ -25,6 +25,12 @@ const (
 	// rootBlockSpans is the number of spans to scan per span
 	// root.
 	rootBlockSpans = 8 * 1024 // 64MB worth of spans
+
+	// maxObletBytes is the maximum bytes of an object to scan at
+	// once. Larger objects will be split up into "oblets" of at
+	// most this size. Since we can scan 1–2 MB/ms, 128 KB bounds
+	// scan preemption at ~100 µs.
+	maxObletBytes = 128 << 10
 )
 
 // gcMarkRootPrepare queues root scanning jobs (stacks, globals, and
@@ -1108,9 +1114,10 @@ func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork) {
 }
 
 // scanobject scans the object starting at b, adding pointers to gcw.
-// b must point to the beginning of a heap object; scanobject consults
-// the GC bitmap for the pointer mask and the spans for the size of the
-// object.
+// b must point to the beginning of a heap object or an oblet.
+// scanobject consults the GC bitmap for the pointer mask and the
+// spans for the size of the object.
+//
 //go:nowritebarrier
 func scanobject(b uintptr, gcw *gcWork) {
 	// Note that arena_used may change concurrently during
@@ -1125,16 +1132,35 @@ func scanobject(b uintptr, gcw *gcWork) {
 	arena_start := mheap_.arena_start
 	arena_used := mheap_.arena_used
 
-	// Find bits of the beginning of the object.
-	// b must point to the beginning of a heap object, so
-	// we can get its bits and span directly.
+	// Find the bits for b and the size of the object at b.
+	//
+	// b is either the beginning of an object, in which case this
+	// is the size of the object to scan, or it points to an
+	// oblet, in which case we compute the size to scan below.
 	hbits := heapBitsForAddr(b)
 	s := spanOfUnchecked(b)
 	n := s.elemsize
 	if n == 0 {
 		throw("scanobject n == 0")
 	}
 
+	if n > maxObletBytes {
+		// Large object. Break into oblets.
+		if b == s.base() {
+			// Enqueue the other oblets for later scans.
+			for oblet := b + maxObletBytes; oblet < s.base()+s.elemsize; oblet += maxObletBytes {
+				if !gcw.putFast(oblet) {
+					gcw.put(oblet)
+				}
+			}
+		}
+		// Compute the size of the oblet.
+		n = s.base() + s.elemsize - b
+		if n > maxObletBytes {
+			n = maxObletBytes
+		}
+	}
+
 	var i uintptr
 	for i = 0; i < n; i += sys.PtrSize {
 		// Find bits for this word.