runtime: track background scan work credit

This tracks scan work done by background GC in a global pool. Mutator
assists will draw on this credit to avoid doing work when background
GC is staying ahead.

Unlike the other GC controller tracking variables, this will be both
written and read throughout the cycle. Hence, we can't arbitrarily
delay updates like we can for scan work and bytes marked. However, we
still want to minimize contention, so this global credit pool is
allowed some error from the "true" amount of credit. Background GC
accumulates credit locally up to a limit and only then flushes to the
global pool. Similarly, mutator assists will draw from the credit pool
in batches.

Change-Id: I1aa4fc604b63bf53d1ee2a967694dffdfc3e255e
Reviewed-on: https://go-review.googlesource.com/8834Reviewed-by: Rick Hudson <rlh@golang.org>

src/runtime/mgc.go

@@ -196,6 +196,13 @@ type gcControllerState struct {
 	// end of the cycle.
 	scanWork int64
 
+	// bgScanCredit is the scan work credit accumulated by the
+	// concurrent background scan. This credit is accumulated by
+	// the background scan and stolen by mutator assists. This is
+	// updated atomically. Updates occur in bounded batches, since
+	// it is both written and read throughout the cycle.
+	bgScanCredit int64
+
 	// workRatioAvg is a moving average of the scan work ratio
 	// (scan work per byte marked).
 	workRatioAvg float64
@@ -205,6 +212,7 @@ type gcControllerState struct {
 // for a new GC cycle.
 func (c *gcControllerState) startCycle() {
 	c.scanWork = 0
+	c.bgScanCredit = 0
 
 	// If this is the first GC cycle or we're operating on a very
 	// small heap, fake heap_marked so it looks like next_gc is
@@ -235,6 +243,13 @@ func (c *gcControllerState) endCycle() {
 	c.workRatioAvg = workRatioWeight*workRatio + (1-workRatioWeight)*c.workRatioAvg
 }
 
+// gcBgCreditSlack is the amount of scan work credit background
+// scanning can accumulate locally before updating
+// gcController.bgScanCredit. Lower values give mutator assists more
+// accurate accounting of background scanning. Higher values reduce
+// memory contention.
+const gcBgCreditSlack = 2000
+
 // Determine whether to initiate a GC.
 // If the GC is already working no need to trigger another one.
 // This should establish a feedback loop where if the GC does not
@@ -440,7 +455,7 @@ func gc(mode int) {
 			tMark = nanotime()
 		}
 		var gcw gcWork
-		gcDrain(&gcw)
+		gcDrain(&gcw, gcBgCreditSlack)
 		gcw.dispose()
 		// Despite the barrier in gcDrain, gcDrainNs may still
 		// be doing work at this point. This is okay because
@@ -649,7 +664,7 @@ func gcMark(start_time int64) {
 	gchelperstart()
 	parfordo(work.markfor)
 	var gcw gcWork
-	gcDrain(&gcw)
+	gcDrain(&gcw, -1)
 	gcw.dispose()
 
 	if work.full != 0 {
@@ -831,7 +846,7 @@ func gchelper() {
 	parfordo(work.markfor)
 	if gcphase != _GCscan {
 		var gcw gcWork
-		gcDrain(&gcw) // blocks in getfull
+		gcDrain(&gcw, -1) // blocks in getfull
 		gcw.dispose()
 	}
 

src/runtime/mgcmark.go

@@ -354,12 +354,23 @@ func scanframeworker(frame *stkframe, unused unsafe.Pointer, gcw *gcWork) {
 
 // gcDrain scans objects in work buffers, blackening grey
 // objects until all work buffers have been drained.
+// If flushScanCredit != -1, gcDrain flushes accumulated scan work
+// credit to gcController.bgScanCredit whenever gcw's local scan work
+// credit exceeds flushScanCredit.
 //go:nowritebarrier
-func gcDrain(gcw *gcWork) {
+func gcDrain(gcw *gcWork, flushScanCredit int64) {
 	if gcphase != _GCmark && gcphase != _GCmarktermination {
 		throw("scanblock phase incorrect")
 	}
 
+	var lastScanFlush, nextScanFlush int64
+	if flushScanCredit != -1 {
+		lastScanFlush = gcw.scanWork
+		nextScanFlush = lastScanFlush + flushScanCredit
+	} else {
+		nextScanFlush = int64(^uint64(0) >> 1)
+	}
+
 	for {
 		// If another proc wants a pointer, give it some.
 		if work.nwait > 0 && work.full == 0 {
@@ -378,6 +389,20 @@ func gcDrain(gcw *gcWork) {
 		// into an empty wbuf in scanobject so there could be
 		// a performance hit as we keep fetching fresh wbufs.
 		scanobject(b, 0, nil, gcw)
+
+		// Flush background scan work credit to the global
+		// account if we've accumulated enough locally so
+		// mutator assists can draw on it.
+		if gcw.scanWork >= nextScanFlush {
+			credit := gcw.scanWork - lastScanFlush
+			xaddint64(&gcController.bgScanCredit, credit)
+			lastScanFlush = gcw.scanWork
+			nextScanFlush = lastScanFlush + flushScanCredit
+		}
+	}
+	if flushScanCredit != -1 {
+		credit := gcw.scanWork - lastScanFlush
+		xaddint64(&gcController.bgScanCredit, credit)
 	}
 	checknocurrentwbuf()
 }