runtime: abstract out assist queue management

This puts all of the assist queue-related code together and makes it
easier to modify how the assist queue works.

Change-Id: Id54e06702bdd5a5dd3fef2ce2c14cd7ca215303c
Reviewed-on: https://go-review.googlesource.com/30700Reviewed-by: Rick Hudson <rlh@golang.org>

src/runtime/mgcmark.go

@@ -538,17 +538,43 @@ retry:
 		// there wasn't enough work to do anyway, so we might
 		// as well let background marking take care of the
 		// work that is available.
+		if !gcParkAssist() {
+			goto retry
+		}
+
+		// At this point either background GC has satisfied
+		// this G's assist debt, or the GC cycle is over.
+	}
+}
+
+// gcWakeAllAssists wakes all currently blocked assists. This is used
+// at the end of a GC cycle. gcBlackenEnabled must be false to prevent
+// new assists from going to sleep after this point.
+func gcWakeAllAssists() {
 	lock(&work.assistQueue.lock)
+	injectglist(work.assistQueue.head.ptr())
+	work.assistQueue.head.set(nil)
+	work.assistQueue.tail.set(nil)
+	unlock(&work.assistQueue.lock)
+}
 
-		// If the GC cycle is over, just return. This is the
-		// likely path if we completed above. We do this
-		// under the lock to prevent a GC cycle from ending
-		// between this check and queuing the assist.
+// gcParkAssist puts the current goroutine on the assist queue and parks.
+//
+// gcParkAssist returns whether the assist is now satisfied. If it
+// returns false, the caller must retry the assist.
+//
+//go:nowritebarrier
+func gcParkAssist() bool {
+	lock(&work.assistQueue.lock)
+	// If the GC cycle finished while we were getting the lock,
+	// exit the assist. The cycle can't finish while we hold the
+	// lock.
 	if atomic.Load(&gcBlackenEnabled) == 0 {
 		unlock(&work.assistQueue.lock)
-			return
+		return true
 	}
 
+	gp := getg()
 	oldHead, oldTail := work.assistQueue.head, work.assistQueue.tail
 	if oldHead == 0 {
 		work.assistQueue.head.set(gp)
@@ -557,6 +583,7 @@ retry:
 	}
 	work.assistQueue.tail.set(gp)
 	gp.schedlink.set(nil)
+
 	// Recheck for background credit now that this G is in
 	// the queue, but can still back out. This avoids a
 	// race in case background marking has flushed more
@@ -568,25 +595,11 @@ retry:
 			oldTail.ptr().schedlink.set(nil)
 		}
 		unlock(&work.assistQueue.lock)
-			goto retry
+		return false
 	}
-		// Park for real.
+	// Park.
 	goparkunlock(&work.assistQueue.lock, "GC assist wait", traceEvGoBlock, 2)
-
-		// At this point either background GC has satisfied
-		// this G's assist debt, or the GC cycle is over.
-	}
-}
-
-// gcWakeAllAssists wakes all currently blocked assists. This is used
-// at the end of a GC cycle. gcBlackenEnabled must be false to prevent
-// new assists from going to sleep after this point.
-func gcWakeAllAssists() {
-	lock(&work.assistQueue.lock)
-	injectglist(work.assistQueue.head.ptr())
-	work.assistQueue.head.set(nil)
-	work.assistQueue.tail.set(nil)
-	unlock(&work.assistQueue.lock)
+	return true
 }
 
 // gcFlushBgCredit flushes scanWork units of background scan work