runtime: Start and stop individual goroutines at gc safepoints

Code to bring goroutines to a gc safepoint one at a time,
do some work such as scanning, and restart the
goroutine, and then move on to the next goroutine.
Currently this code does not do much useful work
but this infrastructure will be critical to future
concurrent GC work.

Fixed comments reviewers.

LGTM=rsc
R=golang-codereviews, rsc, dvyukov
CC=golang-codereviews
https://golang.org/cl/131580043

src/pkg/runtime/mgc0.c

@@ -432,6 +432,7 @@ markroot(ParFor *desc, uint32 i)
 	G *gp;
 	void *p;
 	uint32 status;
+	bool restart;
 
 	USED(&desc);
 	// Note: if you add a case here, please also update heapdump.c:dumproots.
@@ -493,9 +494,15 @@ markroot(ParFor *desc, uint32 i)
 			gp->waitsince = work.tstart;
 		// Shrink a stack if not much of it is being used.
 		runtime·shrinkstack(gp);
+		if(runtime·readgstatus(gp) == Gdead) 
+			gp->gcworkdone = true;
+		else 
+			gp->gcworkdone = false; 
+		restart = runtime·stopg(gp);
 		scanstack(gp);
+		if(restart)
+			runtime·restartg(gp);
 		break;
-		
 	}
 }
 
@@ -687,7 +694,12 @@ scanstack(G *gp)
 	uintptr sp, guard;
 	bool (*fn)(Stkframe*, void*);
 
-	switch(runtime·readgstatus(gp)) {
+	if(runtime·readgstatus(gp)&Gscan == 0) {
+		runtime·printf("runtime: gp=%p, goid=%D, gp->atomicstatus=%d\n", gp, gp->goid, runtime·readgstatus(gp));
+		runtime·throw("mark - bad status");
+	}
+
+	switch(runtime·readgstatus(gp)&~Gscan) {
 	default:
 		runtime·printf("runtime: gp=%p, goid=%D, gp->atomicstatus=%d\n", gp, gp->goid, runtime·readgstatus(gp));
 		runtime·throw("mark - bad status");
@@ -747,6 +759,29 @@ scanstack(G *gp)
 	}
 }
 
+// The gp has been moved to a gc safepoint. If there is gcphase specific
+// work it is done here. 
+void
+runtime·gcphasework(G *gp)
+{
+	switch(runtime·gcphase) {
+	default:
+		runtime·throw("gcphasework in bad gcphase");
+	case GCoff:
+	case GCquiesce:
+	case GCstw:
+	case GCsweep:
+		// No work for now.
+		break;
+	case GCmark:
+		// Disabled until concurrent GC is implemented
+		// but indicate the scan has been done. 
+		// scanstack(gp);
+		break;
+	}
+	gp->gcworkdone = true;
+}
+
 void
 runtime·queuefinalizer(byte *p, FuncVal *fn, uintptr nret, Type *fint, PtrType *ot)
 {

src/pkg/runtime/proc.c

@@ -128,6 +128,7 @@ static bool preemptone(P*);
 static bool exitsyscallfast(void);
 static bool haveexperiment(int8*);
 static void allgadd(G*);
+static void dropg(void);
 
 extern String runtime·buildVersion;
 
@@ -272,7 +273,8 @@ runtime·main(void)
 static void
 dumpgstatus(G* gp)
 {
-	runtime·printf("runtime: gp=%p, goid=%D, gp->atomicstatus=%d\n", gp, gp->goid, runtime·readgstatus(gp));
+	runtime·printf("runtime: gp: gp=%p, goid=%D, gp->atomicstatus=%x\n", gp, gp->goid, runtime·readgstatus(gp));
+	runtime·printf("runtime:  g:  g=%p, goid=%D,  g->atomicstatus=%x\n", g, g->goid, runtime·readgstatus(g));
 }
 
 static void
@@ -508,10 +510,202 @@ runtime·casgstatus(G *gp, uint32 oldval, uint32 newval)
 		runtime·throw("casgstatus: bad incoming values");
 	}
 
-	while(!runtime·cas(&gp->atomicstatus, oldval, newval)) {
 	// loop if gp->atomicstatus is in a scan state giving
 	// GC time to finish and change the state to oldval.
+	while(!runtime·cas(&gp->atomicstatus, oldval, newval)) {
+		// Help GC if needed. 
+		if(gp->preemptscan && !gp->gcworkdone && (oldval == Grunning || oldval == Gsyscall)) {
+			gp->preemptscan = false;
+			runtime·gcphasework(gp);
+		}
+	}	
+}
+
+// stopg ensures that gp is stopped at a GC safe point where its stack can be scanned
+// or in the context of a moving collector the pointers can be flipped from pointing 
+// to old object to pointing to new objects. 
+// If stopg returns true, the caller knows gp is at a GC safe point and will remain there until
+// the caller calls restartg.
+// If stopg returns false, the caller is not responsible for calling restartg. This can happen
+// if another thread, either the gp itself or another GC thread is taking the responsibility 
+// to do the GC work related to this thread.
+bool
+runtime·stopg(G *gp)
+{
+	uint32 s;
+
+	for(;;) {
+		if(gp->gcworkdone)
+			return false;
+
+		s = runtime·readgstatus(gp);
+		switch(s) {
+		default:
+			dumpgstatus(gp);
+			runtime·throw("stopg: gp->atomicstatus is not valid");
+
+		case Gdead:
+			return false;
+
+		case Gcopystack:
+			// Loop until a new stack is in place.
+			break;
+
+		case Grunnable:
+		case Gsyscall:
+		case Gwaiting:
+			// Claim goroutine by setting scan bit.
+			if(!runtime·castogscanstatus(gp, s, s|Gscan))
+				break;
+			// In scan state, do work.
+			runtime·gcphasework(gp);
+			return true;
+
+		case Gscanrunnable:
+		case Gscanwaiting:
+		case Gscansyscall:
+			// Goroutine already claimed by another GC helper.
+			return false;
+
+		case Grunning:
+			// Claim goroutine, so we aren't racing with a status
+			// transition away from Grunning.
+			if(!runtime·castogscanstatus(gp, Grunning, Gscanrunning))
+				break;
+
+			// Mark gp for preemption.
+			if(!gp->gcworkdone) {
+				gp->preemptscan = true;
+				gp->preempt = true;
+				gp->stackguard0 = StackPreempt;
+			}
+
+			// Unclaim.
+			runtime·casfromgscanstatus(gp, Gscanrunning, Grunning);
+			return false;
 		}
+	}
+	// Should not be here....
+}
+
+// The GC requests that this routine be moved from a scanmumble state to a mumble state.
+void 
+runtime·restartg (G *gp)
+{
+	uint32 s;
+
+	s = runtime·readgstatus(gp);
+	switch(s) {
+	default:
+		dumpgstatus(gp); 
+		runtime·throw("restartg: unexpected status");
+
+	case Gdead:
+		break;
+
+	case Gscanrunnable:
+	case Gscanwaiting:
+	case Gscansyscall:
+		runtime·casfromgscanstatus(gp, s, s&~Gscan);
+		break;
+
+	case Gscanenqueue:
+		// Scan is now completed.
+		// Goroutine now needs to be made runnable.
+		// We put it on the global run queue; ready blocks on the global scheduler lock.
+		runtime·casfromgscanstatus(gp, Gscanenqueue, Gwaiting);
+		if(gp != g->m->curg)
+			runtime·throw("processing Gscanenqueue on wrong m");
+		dropg();
+		runtime·ready(gp);
+		break;
+	}
+}
+
+static void
+stopscanstart(G* gp)
+{
+	if(g == gp)
+		runtime·throw("GC not moved to G0");
+	if(runtime·stopg(gp)) {
+		if(!isscanstatus(runtime·readgstatus(gp))) {
+			dumpgstatus(gp);
+			runtime·throw("GC not in scan state");
+		}
+		runtime·restartg(gp);
+	}
+}
+
+// Runs on g0 and does the actual work after putting the g back on the run queue.
+static void
+mquiesce(G *gpmaster)
+{
+	G* gp;
+	uint32 i;
+	uint32 status;
+	uint32 activeglen;
+
+	activeglen = runtime·allglen;
+	// enqueue the calling goroutine.
+	runtime·restartg(gpmaster);
+	for(i = 0; i < activeglen; i++) {
+		gp = runtime·allg[i];
+		if(runtime·readgstatus(gp) == Gdead) 
+			gp->gcworkdone = true; // noop scan.
+		else 
+			gp->gcworkdone = false; 
+		stopscanstart(gp); 
+	}
+
+	// Check that the G's gcwork (such as scanning) has been done. If not do it now. 
+	// You can end up doing work here if the page trap on a Grunning Goroutine has
+	// not been sprung or in some race situations. For example a runnable goes dead
+	// and is started up again with a gp->gcworkdone set to false.
+	for(i = 0; i < activeglen; i++) {
+		gp = runtime·allg[i];
+		while (!gp->gcworkdone) {
+			status = runtime·readgstatus(gp);
+			if(status == Gdead) {
+				gp->gcworkdone = true; // scan is a noop
+				break;
+				//do nothing, scan not needed. 
+			}
+			if(status == Grunning && gp->stackguard0 == (uintptr)StackPreempt && runtime·notetsleep(&runtime·sched.stopnote, 100*1000)) // nanosecond arg 
+				runtime·noteclear(&runtime·sched.stopnote);
+			else 
+				stopscanstart(gp);
+		}
+	}
+
+	for(i = 0; i < activeglen; i++) {
+		gp = runtime·allg[i];
+		status = runtime·readgstatus(gp);
+		if(isscanstatus(status)) {
+			runtime·printf("mstopandscang:bottom: post scan bad status gp=%p has status %x\n", gp, status);
+			dumpgstatus(gp);
+		}
+		if(!gp->gcworkdone && status != Gdead) {
+			runtime·printf("mstopandscang:bottom: post scan gp=%p->gcworkdone still false\n", gp);
+			dumpgstatus(gp);
+		}
+	}
+
+	schedule(); // Never returns.
+}
+
+// quiesce moves all the goroutines to a GC safepoint which for now is a at preemption point.
+// If the global runtime·gcphase is GCmark quiesce will ensure that all of the goroutine's stacks
+// have been scanned before it returns.
+void
+runtime·quiesce(G* mastergp)
+{
+	void (*fn)(G*);
+
+	runtime·castogscanstatus(mastergp, Grunning, Gscanenqueue);
+	// Now move this to the g0 (aka m) stack.
+	// g0 will potentially scan this thread and put mastergp on the runqueue 
+	fn = mquiesce;
+	runtime·mcall(&fn);
 }
 
 // This is used by the GC as well as the routines that do stack dumps. In the case
@@ -1503,7 +1697,7 @@ runtime·gosched_m(G *gp)
 	uint32 status;
 
 	status = runtime·readgstatus(gp);
-	if ((status&~Gscan) != Grunning){
+	if((status&~Gscan) != Grunning){
 		dumpgstatus(gp);
 		runtime·throw("bad g status");
 	}
@@ -2031,7 +2225,7 @@ allgadd(G *gp)
 	G **new;
 	uintptr cap;
 
-	if (runtime·readgstatus(gp) == Gidle) 
+	if(runtime·readgstatus(gp) == Gidle) 
 		runtime·throw("allgadd: bad status Gidle");
 
 	runtime·lock(&allglock);
@@ -2062,7 +2256,7 @@ gfput(P *p, G *gp)
 	uintptr stksize;
 	Stktop *top;
 
-	if (runtime·readgstatus(gp) != Gdead) 
+	if(runtime·readgstatus(gp) != Gdead) 
 		runtime·throw("gfput: bad status (not Gdead)");
 
 	if(gp->stackguard - StackGuard != gp->stack0)

src/pkg/runtime/runtime.h

@@ -292,7 +292,7 @@ struct	G
 	bool	preempt;	// preemption signal, duplicates stackguard0 = StackPreempt
 	bool	paniconfault;	// panic (instead of crash) on unexpected fault address
 	bool    preemptscan;    // preempted g does scan for GC
-	bool    scancheck;      // debug: cleared at begining of scan cycle, set by scan, tested at end of cycle
+	bool    gcworkdone;     // debug: cleared at begining of gc work phase cycle, set by gcphasework, tested at end of cycle
 	int8	raceignore;	// ignore race detection events
 	M*	m;		// for debuggers, but offset not hard-coded
 	M*	lockedm;
@@ -579,6 +579,16 @@ struct DebugVars
 	int32	scavenge;
 };
 
+// Indicates to write barrier and sychronization task to preform.
+enum
+{                   // Synchronization            Write barrier
+	GCoff,      // stop and start             nop
+	GCquiesce,  // stop and start             nop
+	GCstw,      // stop the ps                nop
+	GCmark,     // scan the stacks and start  no white to black
+	GCsweep,    // stop and start             nop
+};
+
 struct ForceGCState
 {
 	Mutex	lock;
@@ -586,6 +596,7 @@ struct ForceGCState
 	uint32	idle;
 };
 
+extern uint32 runtime·gcphase;
 extern bool runtime·precisestack;
 extern bool runtime·copystack;
 
@@ -614,8 +625,12 @@ enum {
 	HashRandomBytes = 32
 };
 
-uint32  runtime·readgstatus(G *gp);
+uint32  runtime·readgstatus(G*);
 void    runtime·casgstatus(G*, uint32, uint32);
+void    runtime·quiesce(G*);
+bool    runtime·stopg(G*);
+void    runtime·restartg(G*);
+void    runtime·gcphasework(G*);
 
 /*
  * deferred subroutine calls

src/pkg/runtime/stack.c

@@ -936,6 +936,14 @@ runtime·newstack(void)
 			runtime·throw("runtime: g is running but p is not");
 		if(oldstatus == Gsyscall && g->m->locks == 0)
 			runtime·throw("runtime: stack growth during syscall");
+		if(oldstatus == Grunning && gp->preemptscan) {
+			runtime·gcphasework(gp);
+			runtime·casgstatus(gp, Gwaiting, Grunning);
+			gp->stackguard0 = gp->stackguard;
+			gp->preempt = false; 
+			gp->preemptscan = false;        // Tells the GC premption was successful.
+			runtime·gogo(&gp->sched);	// never return 
+		}
 
 		// Be conservative about where we preempt.
 		// We are interested in preempting user Go code, not runtime code.