runtime: changes to g->atomicstatus (nee status) to support concurrent GC

Every change to g->atomicstatus is now done atomically so that we can
ensure that all gs pass through a gc safepoint on demand. This allows
the GC to move from one phase to the next safely. In some phases the
stack will be scanned. This CL only deals with the infrastructure that
allows g->atomicstatus to go from one state to another. Future CLs
will deal with scanning and monitoring what phase the GC is in.

The major change was to moving to using a Gscan bit to indicate that
the status is in a scan state. The only bug fix was in oldstack where
I wasn't moving to a Gcopystack state in order to block scanning until
the new stack was in place. The proc.go file is waiting for an atomic
load instruction.

LGTM=rsc
R=golang-codereviews, dvyukov, josharian, rsc
CC=golang-codereviews, khr
https://golang.org/cl/132960044

src/pkg/runtime/heapdump.c

@@ -396,7 +396,7 @@ dumpgoroutine(G *gp)
 	dumpint((uintptr)sp);
 	dumpint(gp->goid);
 	dumpint(gp->gopc);
-	dumpint(gp->status);
+	dumpint(runtime·readgstatus(gp));
 	dumpbool(gp->issystem);
 	dumpbool(false);  // isbackground
 	dumpint(gp->waitsince);
@@ -442,14 +442,16 @@ dumpgs(void)
 {
 	G *gp;
 	uint32 i;
+	uint32 status;
 
 	// goroutines & stacks
 	for(i = 0; i < runtime·allglen; i++) {
 		gp = runtime·allg[i];
-		switch(gp->status){
+		status = runtime·readgstatus(gp); // The world is stopped so gp will not be in a scan state.
+		switch(status){
 		default:
-			runtime·printf("unexpected G.status %d\n", gp->status);
-			runtime·throw("mark - bad status");
+			runtime·printf("runtime: unexpected G.status %d\n", status);
+			runtime·throw("dumpgs in STW - bad status");
 		case Gdead:
 			break;
 		case Grunnable:
@@ -730,7 +732,7 @@ mdump(G *gp)
 	flush();
 
 	gp->param = nil;
-	gp->status = Grunning;
+	runtime·casgstatus(gp, Gwaiting, Grunning);
 	runtime·gogo(&gp->sched);
 }
 
@@ -751,7 +753,7 @@ runtime∕debug·WriteHeapDump(uintptr fd)
 	dumpfd = fd;
 
 	// Call dump routine on M stack.
-	g->status = Gwaiting;
+	runtime·casgstatus(g, Grunning, Gwaiting);
 	g->waitreason = runtime·gostringnocopy((byte*)"dumping heap");
 	runtime·mcall(mdump);
 

src/pkg/runtime/mgc0.c

@@ -484,6 +484,7 @@ markroot(ParFor *desc, uint32 i)
 	uint32 spanidx, sg;
 	G *gp;
 	void *p;
+	uint32 status;
 
 	USED(&desc);
 	// Note: if you add a case here, please also update heapdump.c:dumproots.
@@ -540,7 +541,8 @@ markroot(ParFor *desc, uint32 i)
 		gp = runtime·allg[i - RootCount];
 		// remember when we've first observed the G blocked
 		// needed only to output in traceback
-		if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->waitsince == 0)
+		status = runtime·readgstatus(gp);
+		if((status == Gwaiting || status == Gsyscall) && gp->waitsince == 0)
 			gp->waitsince = work.tstart;
 		// Shrink a stack if not much of it is being used.
 		runtime·shrinkstack(gp);
@@ -737,13 +739,14 @@ scanstack(G *gp)
 	Stktop *stk;
 	uintptr sp, guard;
 
-	switch(gp->status){
+	switch(runtime·readgstatus(gp)) {
 	default:
-		runtime·printf("unexpected G.status %d (goroutine %p %D)\n", gp->status, gp, gp->goid);
+		runtime·printf("runtime: gp=%p, goid=%D, gp->atomicstatus=%d\n", gp, gp->goid, runtime·readgstatus(gp));
 		runtime·throw("mark - bad status");
 	case Gdead:
 		return;
 	case Grunning:
+		runtime·printf("runtime: gp=%p, goid=%D, gp->atomicstatus=%d\n", gp, gp->goid, runtime·readgstatus(gp));
 		runtime·throw("mark - world not stopped");
 	case Grunnable:
 	case Gsyscall:
@@ -860,7 +863,7 @@ runtime·MSpan_EnsureSwept(MSpan *s)
 	}
 	// unfortunate condition, and we don't have efficient means to wait
 	while(runtime·atomicload(&s->sweepgen) != sg)
-		runtime·osyield();  
+		runtime·osyield();
 }
 
 // Sweep frees or collects finalizers for blocks not marked in the mark phase.
@@ -1349,7 +1352,7 @@ runtime·gc(int32 force)
 			a.start_time = runtime·nanotime();
 		// switch to g0, call gc(&a), then switch back
 		g->param = &a;
-		g->status = Gwaiting;
+		runtime·casgstatus(g, Grunning, Gwaiting);
 		g->waitreason = runtime·gostringnocopy((byte*)"garbage collection");
 		runtime·mcall(mgc);
 	}
@@ -1373,7 +1376,7 @@ mgc(G *gp)
 {
 	gc(gp->param);
 	gp->param = nil;
-	gp->status = Grunning;
+	runtime·casgstatus(gp, Gwaiting, Grunning);
 	runtime·gogo(&gp->sched);
 }
 
@@ -1384,14 +1387,14 @@ runtime·gc_m(void)
 	G *gp;
 
 	gp = g->m->curg;
-	gp->status = Gwaiting;
+	runtime·casgstatus(gp, Grunning, Gwaiting);
 	gp->waitreason = runtime·gostringnocopy((byte*)"garbage collection");
 
 	a.start_time = (uint64)(g->m->scalararg[0]) | ((uint64)(g->m->scalararg[1]) << 32);
 	a.eagersweep = g->m->scalararg[2];
 	gc(&a);
 
-	gp->status = Grunning;
+	runtime·casgstatus(gp, Gwaiting, Grunning);
 }
 
 static void

src/pkg/runtime/mprof.goc

@@ -288,7 +288,7 @@ func GoroutineProfile(b Slice) (n int, ok bool) {
 			saveg(pc, sp, g, r++);
 			for(i = 0; i < runtime·allglen; i++) {
 				gp = runtime·allg[i];
-				if(gp == g || gp->status == Gdead)
+				if(gp == g || runtime·readgstatus(gp) == Gdead)
 					continue;
 				saveg(~(uintptr)0, ~(uintptr)0, gp, r++);
 			}

src/pkg/runtime/panic.c

@@ -477,6 +477,7 @@ bool
 runtime·canpanic(G *gp)
 {
 	M *m;
+	uint32 status;
 
 	// Note that g is m->gsignal, different from gp.
 	// Note also that g->m can change at preemption, so m can go stale
@@ -490,7 +491,8 @@ runtime·canpanic(G *gp)
 		return false;
 	if(m->locks-m->softfloat != 0 || m->mallocing != 0 || m->throwing != 0 || m->gcing != 0 || m->dying != 0)
 		return false;
-	if(gp->status != Grunning || gp->syscallsp != 0)
+	status = runtime·readgstatus(gp);
+	if((status&~Gscan) != Grunning || gp->syscallsp != 0)
 		return false;
 #ifdef GOOS_windows
 	if(m->libcallsp != 0)

src/pkg/runtime/proc.go

@@ -6,6 +6,9 @@ package runtime
 
 import "unsafe"
 
+// This is not mechanically generated
+// so be very careful and refer to runtime.h
+// for the definitive enum.
 const (
 	gStatusidle = iota
 	gStatusRunnable
@@ -14,6 +17,14 @@ const (
 	gStatusWaiting
 	gStatusMoribundUnused
 	gStatusDead
+	gStatusEnqueue
+	gStatusCopystack
+	gStatusScan         = 0x1000
+	gStatusScanRunnable = gStatusScan + gStatusRunnable
+	gStatusScanRunning  = gStatusScan + gStatusRunning
+	gStatusScanSyscall  = gStatusScan + gStatusSyscall
+	gStatusScanWaiting  = gStatusScan + gStatusWaiting
+	gStatusScanEnqueue  = gStatusScan + gStatusEnqueue
 )
 
 var parkunlock_c byte
@@ -24,12 +35,18 @@ func Gosched() {
 	mcall(&gosched_m)
 }
 
+func readgStatus(gp *g) uint32 {
+	//return atomic.LoadUint32(&gp.atomicstatus) // TODO: add bootstrap code to provide.
+	return gp.atomicstatus
+}
+
 // Puts the current goroutine into a waiting state and calls unlockf.
 // If unlockf returns false, the goroutine is resumed.
 func gopark(unlockf unsafe.Pointer, lock unsafe.Pointer, reason string) {
 	mp := acquirem()
 	gp := mp.curg
-	if gp.status != gStatusRunning {
+	status := readgStatus(gp)
+	if status != gStatusRunning && status != gStatusScanRunning {
 		gothrow("gopark: bad g status")
 	}
 	mp.waitlock = lock

src/pkg/runtime/runtime.h

@@ -126,13 +126,25 @@ enum
 	// If you add to this list, add to the list
 	// of "okay during garbage collection" status
 	// in mgc0.c too.
-	Gidle,
-	Grunnable,
-	Grunning,
-	Gsyscall,
-	Gwaiting,
-	Gmoribund_unused,  // currently unused, but hardcoded in gdb scripts
-	Gdead,
+	Gidle,                                 // 0
+	Grunnable,                             // 1 runnable and on a run queue
+	Grunning,                              // 2
+	Gsyscall,                              // 3
+	Gwaiting,                              // 4
+	Gmoribund_unused,                      // 5 currently unused, but hardcoded in gdb scripts
+	Gdead,                                 // 6
+	Genqueue,                              // 7 Only the Gscanenqueue is used.
+	Gcopystack,                            // 8 in this state when newstack is moving the stack
+	// the following encode that the GC is scanning the stack and what to do when it is done 
+	Gscan = 0x1000,                        // atomicstatus&~Gscan = the non-scan state,
+	// Gscanidle =     Gscan + Gidle,      // Not used. Gidle only used with newly malloced gs
+	Gscanrunnable = Gscan + Grunnable,     //  0x1001 When scanning complets make Grunnable (it is already on run queue)
+	Gscanrunning =  Gscan + Grunning,      //  0x1002 Used to tell preemption newstack routine to scan preempted stack.
+	Gscansyscall =  Gscan + Gsyscall,      //  0x1003 When scanning completes make is Gsyscall
+	Gscanwaiting =  Gscan + Gwaiting,      //  0x1004 When scanning completes make it Gwaiting
+	// Gscanmoribund_unused,               //  not possible
+	// Gscandead,                          //  not possible
+	Gscanenqueue = Gscan + Genqueue,       //  When scanning completes make it Grunnable and put on runqueue
 };
 enum
 {
@@ -276,7 +288,7 @@ struct	G
 	uintptr	stack0;
 	uintptr	stacksize;
 	void*	param;		// passed parameter on wakeup
-	int16	status;
+	uint32	atomicstatus;
 	int64	goid;
 	int64	waitsince;	// approx time when the G become blocked
 	String	waitreason;	// if status==Gwaiting
@@ -285,6 +297,8 @@ struct	G
 	bool	issystem;	// do not output in stack dump, ignore in deadlock detector
 	bool	preempt;	// preemption signal, duplicates stackguard0 = StackPreempt
 	bool	paniconfault;	// panic (instead of crash) on unexpected fault address
+	bool    preemptscan;    // preempted g does scan for GC
+	bool    scancheck;      // debug: cleared at begining of scan cycle, set by scan, tested at end of cycle
 	int8	raceignore;	// ignore race detection events
 	M*	m;		// for debuggers, but offset not hard-coded
 	M*	lockedm;
@@ -681,6 +695,9 @@ void	runtime·algslicecopy(uintptr, void*, void*);
 void	runtime·intercopy(uintptr, void*, void*);
 void	runtime·nilintercopy(uintptr, void*, void*);
 
+uint32  runtime·readgstatus(G *gp);
+void    runtime·casgstatus(G*, uint32, uint32);
+
 /*
  * deferred subroutine calls
  */

src/pkg/runtime/stack.c

@@ -337,15 +337,20 @@ runtime·oldstack(void)
 			top->gobuf.pc, top->gobuf.sp, top->gobuf.lr, (uintptr)g->m->cret, (uintptr)argsize);
 	}
 
-	// gp->status is usually Grunning, but it could be Gsyscall if a stack overflow
-	// happens during a function call inside entersyscall.
-	oldstatus = gp->status;
-	
 	gp->sched = top->gobuf;
 	gp->sched.ret = g->m->cret;
 	g->m->cret = 0; // drop reference
-	gp->status = Gwaiting;
-	gp->waitreason = runtime·gostringnocopy((byte*)"stack unsplit");
+	// gp->status is usually Grunning, but it could be Gsyscall if a stack overflow
+	// happens during a function call inside entersyscall.
+
+	oldstatus = runtime·readgstatus(gp);
+	oldstatus &= ~Gscan;
+	if(oldstatus != Grunning && oldstatus != Gsyscall) {
+		runtime·printf("runtime: oldstack status=%d\n", oldstatus);
+		runtime·throw("oldstack");
+	}
+	runtime·casgstatus(gp, oldstatus, Gcopystack);
+	gp->waitreason = runtime·gostringnocopy((byte*)"stack unsplit");	
 
 	if(argsize > 0) {
 		sp -= argsize;
@@ -363,8 +368,7 @@ runtime·oldstack(void)
 	gp->stackguard0 = gp->stackguard;
 	gp->panicwrap = top->panicwrap;
 	runtime·stackfree(gp, old, top);
-
-	gp->status = oldstatus;
+	runtime·casgstatus(gp, Gcopystack, oldstatus); // oldstatus is Grunning or Gsyscall
 	runtime·gogo(&gp->sched);
 }
 
@@ -768,6 +772,7 @@ copystack(G *gp, uintptr nframes, uintptr newsize)
 	uintptr oldsize, used;
 	AdjustInfo adjinfo;
 	Stktop *oldtop, *newtop;
+	uint32 oldstatus;
 
 	if(gp->syscallstack != 0)
 		runtime·throw("can't handle stack copy in syscall yet");
@@ -801,7 +806,12 @@ copystack(G *gp, uintptr nframes, uintptr newsize)
 	
 	// copy the stack (including Stktop) to the new location
 	runtime·memmove(newbase - used, oldbase - used, used);
-	
+	oldstatus = runtime·readgstatus(gp);
+	oldstatus &= ~Gscan;
+	if (oldstatus == Gwaiting || oldstatus == Grunnable)
+		runtime·casgstatus(gp, oldstatus, Gcopystack); // oldstatus is Gwaiting or Grunnable
+	else
+		runtime·throw("copystack: bad status, not Gwaiting or Grunnable");
 	// Swap out old stack for new one
 	gp->stackbase = (uintptr)newtop;
 	gp->stackguard = (uintptr)newstk + StackGuard;
@@ -810,6 +820,8 @@ copystack(G *gp, uintptr nframes, uintptr newsize)
 		gp->stack0 = (uintptr)newstk;
 	gp->sched.sp = (uintptr)(newbase - used);
 
+	runtime·casgstatus(gp, Gcopystack, oldstatus); // oldstatus is Gwaiting or Grunnable
+
 	// free old stack
 	runtime·stackfree(gp, oldstk, oldtop);
 }
@@ -831,6 +843,9 @@ runtime·round2(int32 x)
 // m->moreframesize bytes, copy m->moreargsize bytes to the new frame,
 // and then act as though runtime·lessstack called the function at
 // m->morepc.
+//
+// g->atomicstatus will be Grunning, Gsyscall or Gscanrunning, Gscansyscall upon entry. 
+// If the GC is trying to stop this g then it will set preemptscan to true.
 void
 runtime·newstack(void)
 {
@@ -853,11 +868,13 @@ runtime·newstack(void)
 		runtime·throw("runtime: wrong goroutine in newstack");
 	}
 
+	// The goroutine must be executing in order to call newstack, so the possible states are
+	// Grunning and Gsyscall (and, due to GC, also Gscanrunning and Gscansyscall).	
+
 	// gp->status is usually Grunning, but it could be Gsyscall if a stack overflow
 	// happens during a function call inside entersyscall.
 	gp = g->m->curg;
-	oldstatus = gp->status;
-
+	oldstatus = runtime·readgstatus(gp) & ~Gscan;
 	framesize = g->m->moreframesize;
 	argsize = g->m->moreargsize;
 	moreargp = g->m->moreargp;
@@ -866,7 +883,8 @@ runtime·newstack(void)
 	g->m->morebuf.pc = (uintptr)nil;
 	g->m->morebuf.lr = (uintptr)nil;
 	g->m->morebuf.sp = (uintptr)nil;
-	gp->status = Gwaiting;
+
+	runtime·casgstatus(gp, oldstatus, Gwaiting); // oldstatus is not in a Gscan status
 	gp->waitreason = runtime·gostringnocopy((byte*)"stack growth");
 	newstackcall = framesize==1;
 	if(newstackcall)
@@ -892,6 +910,7 @@ runtime·newstack(void)
 			gp->sched.pc, gp->sched.sp, gp->sched.lr, gp->sched.ctxt);
 	}
 	if(sp < gp->stackguard - StackGuard) {
+		runtime·printf("runtime: gp=%p, gp->status=%d, oldstatus=%d\n ", (void*)gp, runtime·readgstatus(gp), oldstatus);
 		runtime·printf("runtime: split stack overflow: %p < %p\n", sp, gp->stackguard - StackGuard);
 		runtime·throw("runtime: split stack overflow");
 	}
@@ -908,17 +927,18 @@ runtime·newstack(void)
 			runtime·throw("runtime: g is running but p is not");
 		if(oldstatus == Gsyscall && g->m->locks == 0)
 			runtime·throw("runtime: stack growth during syscall");
+
 		// Be conservative about where we preempt.
 		// We are interested in preempting user Go code, not runtime code.
 		if(oldstatus != Grunning || g->m->locks || g->m->mallocing || g->m->gcing || g->m->p->status != Prunning) {
 			// Let the goroutine keep running for now.
 			// gp->preempt is set, so it will be preempted next time.
 			gp->stackguard0 = gp->stackguard;
-			gp->status = oldstatus;
+			runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Gsyscall or Grunning
 			runtime·gogo(&gp->sched);	// never return
 		}
 		// Act like goroutine called runtime.Gosched.
-		gp->status = oldstatus;
+		runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Gsyscall or Grunning
 		runtime·gosched_m(gp);	// never return
 	}
 
@@ -933,6 +953,8 @@ runtime·newstack(void)
 			oldbase = (byte*)gp->stackbase + sizeof(Stktop);
 			oldsize = oldbase - oldstk;
 			newsize = oldsize * 2;
+			// Note that the concurrent GC might be scanning the stack as we try to replace it.
+			// copystack takes care of the appropriate coordination with the stack scanner.
 			copystack(gp, nframes, newsize);
 			if(StackDebug >= 1)
 				runtime·printf("stack grow done\n");
@@ -940,7 +962,7 @@ runtime·newstack(void)
 				runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize);
 				runtime·throw("stack overflow");
 			}
-			gp->status = oldstatus;
+			runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Gsyscall or Grunning
 			runtime·gogo(&gp->sched);
 		}
 		// TODO: if stack is uncopyable because we're in C code, patch return value at
@@ -1017,7 +1039,7 @@ runtime·newstack(void)
 		runtime·gostartcall(&label, (void(*)(void))gp->sched.pc, gp->sched.ctxt);
 		gp->sched.ctxt = nil;
 	}
-	gp->status = oldstatus;
+	runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Grunning or Gsyscall
 	runtime·gogo(&label);
 
 	*(int32*)345 = 123;	// never return
@@ -1055,7 +1077,7 @@ runtime·shrinkstack(G *gp)
 
 	if(!runtime·copystack)
 		return;
-	if(gp->status == Gdead)
+	if(runtime·readgstatus(gp) == Gdead)
 		return;
 	if(gp->stackbase == 0)
 		runtime·throw("stackbase == 0");

src/pkg/runtime/traceback_arm.c

@@ -327,7 +327,7 @@ runtime·traceback(uintptr pc, uintptr sp, uintptr lr, G *gp)
 {
 	int32 n;
 
-	if(gp->status == Gsyscall) {
+	if((runtime·readgstatus(gp)&~Gscan) == Gsyscall){
 		// Override signal registers if blocked in system call.
 		pc = gp->syscallpc;
 		sp = gp->syscallsp;

src/pkg/runtime/traceback_x86.c

@@ -402,7 +402,7 @@ runtime·traceback(uintptr pc, uintptr sp, uintptr lr, G *gp)
 
 	USED(lr);
 
-	if(gp->status == Gsyscall) {
+	if((runtime·readgstatus(gp)&~Gscan) == Gsyscall){
 		// Override signal registers if blocked in system call.
 		pc = gp->syscallpc;
 		sp = gp->syscallsp;