runtime: optimize bulkBarrierPreWrite with allocheaders

Currently bulkBarrierPreWrite follows a fairly slow path wherein it
calls typePointersOf, which ends up calling into fastForward. This does
some fairly heavy computation to move the iterator forward without any
assumptions about where it lands at all. It needs to be completely
general to support splitting at arbitrary boundaries, for example for
scanning oblets.

This means that copying objects during the GC mark phase is fairly
expensive, and is a regression from before allocheaders.

However, in almost all cases bulkBarrierPreWrite and
bulkBarrierPreWriteSrcOnly have perfect type information. We can do a
lot better in these cases because we're starting on a type-size
boundary, which is exactly what the iterator is built around.

This change adds the typePointersOfType method which produces a
typePointers iterator from a pointer and a type. This change
significantly improves the performance of these bulk write barriers,
eliminating some performance regressions that were noticed on the perf
dashboard.

There are still just a couple cases where we have to use the more
general typePointersOf calls, but they're fairly rare; most bulk
barriers have perfect type information.

This change is tested by the GCInfo tests in the runtime and the GCBits
tests in the reflect package via an additional check in getgcmask.

Results for tile38 before and after allocheaders. There was previous a
regression in the p90, now it's gone. Also, the overall win has been
boosted slightly.

tile38 $ benchstat noallocheaders.results allocheaders.results
name             old time/op            new time/op            delta
Tile38QueryLoad             481µs ± 1%             468µs ± 1%  -2.71%  (p=0.000 n=10+10)

name             old average-RSS-bytes  new average-RSS-bytes  delta
Tile38QueryLoad            6.32GB ± 1%            6.23GB ± 0%  -1.38%  (p=0.000 n=9+8)

name             old peak-RSS-bytes     new peak-RSS-bytes     delta
Tile38QueryLoad            6.49GB ± 1%            6.40GB ± 1%  -1.38%  (p=0.002 n=10+10)

name             old peak-VM-bytes      new peak-VM-bytes      delta
Tile38QueryLoad            7.72GB ± 1%            7.64GB ± 1%  -1.07%  (p=0.007 n=10+10)

name             old p50-latency-ns     new p50-latency-ns     delta
Tile38QueryLoad              212k ± 1%              205k ± 0%  -3.02%  (p=0.000 n=10+9)

name             old p90-latency-ns     new p90-latency-ns     delta
Tile38QueryLoad              622k ± 1%              616k ± 1%  -1.03%  (p=0.005 n=10+10)

name             old p99-latency-ns     new p99-latency-ns     delta
Tile38QueryLoad             4.55M ± 2%             4.39M ± 2%  -3.51%  (p=0.000 n=10+10)

name             old ops/s              new ops/s              delta
Tile38QueryLoad             12.5k ± 1%             12.8k ± 1%  +2.78%  (p=0.000 n=10+10)

Change-Id: I0a48f848eae8777d0fd6769c3a1fe449f8d9d0a6
Reviewed-on: https://go-review.googlesource.com/c/go/+/542219
Reviewed-by: Cherry Mui <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: mknyszek

src/reflect/value.go

@@ -1723,7 +1723,7 @@ func (v Value) SetZero() {
 	case Slice:
 		*(*unsafeheader.Slice)(v.ptr) = unsafeheader.Slice{}
 	case Interface:
-		*(*[2]unsafe.Pointer)(v.ptr) = [2]unsafe.Pointer{}
+		*(*emptyInterface)(v.ptr) = emptyInterface{}
 	case Chan, Func, Map, Pointer, UnsafePointer:
 		*(*unsafe.Pointer)(v.ptr) = nil
 	case Array, Struct:

src/runtime/mbarrier.go

@@ -154,7 +154,10 @@ func typedmemmove(typ *abi.Type, dst, src unsafe.Pointer) {
 		return
 	}
 	if writeBarrier.enabled && typ.PtrBytes != 0 {
-		bulkBarrierPreWrite(uintptr(dst), uintptr(src), typ.PtrBytes)
+		// This always copies a full value of type typ so it's safe
+		// to pass typ along as an optimization. See the comment on
+		// bulkBarrierPreWrite.
+		bulkBarrierPreWrite(uintptr(dst), uintptr(src), typ.PtrBytes, typ)
 	}
 	// There's a race here: if some other goroutine can write to
 	// src, it may change some pointer in src after we've
@@ -176,7 +179,10 @@ func typedmemmove(typ *abi.Type, dst, src unsafe.Pointer) {
 //go:nowritebarrierrec
 //go:nosplit
 func wbZero(typ *_type, dst unsafe.Pointer) {
-	bulkBarrierPreWrite(uintptr(dst), 0, typ.PtrBytes)
+	// This always copies a full value of type typ so it's safe
+	// to pass typ along as an optimization. See the comment on
+	// bulkBarrierPreWrite.
+	bulkBarrierPreWrite(uintptr(dst), 0, typ.PtrBytes, typ)
 }
 
 // wbMove performs the write barrier operations necessary before
@@ -186,7 +192,11 @@ func wbZero(typ *_type, dst unsafe.Pointer) {
 //go:nowritebarrierrec
 //go:nosplit
 func wbMove(typ *_type, dst, src unsafe.Pointer) {
-	bulkBarrierPreWrite(uintptr(dst), uintptr(src), typ.PtrBytes)
+	// This always copies a full value of type typ so it's safe to
+	// pass a type here.
+	//
+	// See the comment on bulkBarrierPreWrite.
+	bulkBarrierPreWrite(uintptr(dst), uintptr(src), typ.PtrBytes, typ)
 }
 
 //go:linkname reflect_typedmemmove reflect.typedmemmove
@@ -223,7 +233,10 @@ func reflectlite_typedmemmove(typ *_type, dst, src unsafe.Pointer) {
 //go:nosplit
 func reflectcallmove(typ *_type, dst, src unsafe.Pointer, size uintptr, regs *abi.RegArgs) {
 	if writeBarrier.enabled && typ != nil && typ.PtrBytes != 0 && size >= goarch.PtrSize {
-		bulkBarrierPreWrite(uintptr(dst), uintptr(src), size)
+		// Pass nil for the type. dst does not point to value of type typ,
+		// but rather points into one, so applying the optimization is not
+		// safe. See the comment on this function.
+		bulkBarrierPreWrite(uintptr(dst), uintptr(src), size, nil)
 	}
 	memmove(dst, src, size)
 
@@ -278,8 +291,11 @@ func typedslicecopy(typ *_type, dstPtr unsafe.Pointer, dstLen int, srcPtr unsafe
 	// before calling typedslicecopy.
 	size := uintptr(n) * typ.Size_
 	if writeBarrier.enabled {
+		// This always copies one or more full values of type typ so
+		// it's safe to pass typ along as an optimization. See the comment on
+		// bulkBarrierPreWrite.
 		pwsize := size - typ.Size_ + typ.PtrBytes
-		bulkBarrierPreWrite(uintptr(dstPtr), uintptr(srcPtr), pwsize)
+		bulkBarrierPreWrite(uintptr(dstPtr), uintptr(srcPtr), pwsize, typ)
 	}
 	// See typedmemmove for a discussion of the race between the
 	// barrier and memmove.
@@ -308,7 +324,10 @@ func reflect_typedslicecopy(elemType *_type, dst, src slice) int {
 //go:nosplit
 func typedmemclr(typ *_type, ptr unsafe.Pointer) {
 	if writeBarrier.enabled && typ.PtrBytes != 0 {
-		bulkBarrierPreWrite(uintptr(ptr), 0, typ.PtrBytes)
+		// This always clears a whole value of type typ, so it's
+		// safe to pass a type here and apply the optimization.
+		// See the comment on bulkBarrierPreWrite.
+		bulkBarrierPreWrite(uintptr(ptr), 0, typ.PtrBytes, typ)
 	}
 	memclrNoHeapPointers(ptr, typ.Size_)
 }
@@ -321,7 +340,11 @@ func reflect_typedmemclr(typ *_type, ptr unsafe.Pointer) {
 //go:linkname reflect_typedmemclrpartial reflect.typedmemclrpartial
 func reflect_typedmemclrpartial(typ *_type, ptr unsafe.Pointer, off, size uintptr) {
 	if writeBarrier.enabled && typ.PtrBytes != 0 {
-		bulkBarrierPreWrite(uintptr(ptr), 0, size)
+		// Pass nil for the type. ptr does not point to value of type typ,
+		// but rather points into one so it's not safe to apply the optimization.
+		// See the comment on this function in the reflect package and the
+		// comment on bulkBarrierPreWrite.
+		bulkBarrierPreWrite(uintptr(ptr), 0, size, nil)
 	}
 	memclrNoHeapPointers(ptr, size)
 }
@@ -330,7 +353,9 @@ func reflect_typedmemclrpartial(typ *_type, ptr unsafe.Pointer, off, size uintpt
 func reflect_typedarrayclear(typ *_type, ptr unsafe.Pointer, len int) {
 	size := typ.Size_ * uintptr(len)
 	if writeBarrier.enabled && typ.PtrBytes != 0 {
-		bulkBarrierPreWrite(uintptr(ptr), 0, size)
+		// This always clears whole elements of an array, so it's
+		// safe to pass a type here. See the comment on bulkBarrierPreWrite.
+		bulkBarrierPreWrite(uintptr(ptr), 0, size, typ)
 	}
 	memclrNoHeapPointers(ptr, size)
 }
@@ -342,6 +367,7 @@ func reflect_typedarrayclear(typ *_type, ptr unsafe.Pointer, len int) {
 //
 //go:nosplit
 func memclrHasPointers(ptr unsafe.Pointer, n uintptr) {
-	bulkBarrierPreWrite(uintptr(ptr), 0, n)
+	// Pass nil for the type since we don't have one here anyway.
+	bulkBarrierPreWrite(uintptr(ptr), 0, n, nil)
 	memclrNoHeapPointers(ptr, n)
 }