runtime: improved scheduler

Distribute runnable queues, memory cache
and cache of dead G's per processor.
Faster non-blocking syscall enter/exit.
More conservative worker thread blocking/unblocking.

R=dave, bradfitz, remyoudompheng, rsc
CC=golang-dev
https://golang.org/cl/7314062

src/pkg/runtime/mgc0.c

@@ -1633,20 +1633,12 @@ runtime·gchelper(void)
 // extra memory used).
 static int32 gcpercent = GcpercentUnknown;
 
-static void
-stealcache(void)
-{
-	M *mp;
-
-	for(mp=runtime·allm; mp; mp=mp->alllink)
-		runtime·MCache_ReleaseAll(mp->mcache);
-}
-
 static void
 cachestats(GCStats *stats)
 {
 	M *mp;
 	MCache *c;
+	P *p, **pp;
 	int32 i;
 	uint64 stacks_inuse;
 	uint64 *src, *dst;
@@ -1655,8 +1647,6 @@ cachestats(GCStats *stats)
 		runtime·memclr((byte*)stats, sizeof(*stats));
 	stacks_inuse = 0;
 	for(mp=runtime·allm; mp; mp=mp->alllink) {
-		c = mp->mcache;
-		runtime·purgecachedstats(c);
 		stacks_inuse += mp->stackinuse*FixedStack;
 		if(stats) {
 			src = (uint64*)&mp->gcstats;
@@ -1665,6 +1655,12 @@ cachestats(GCStats *stats)
 				dst[i] += src[i];
 			runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats));
 		}
+	}
+	for(pp=runtime·allp; p=*pp; pp++) {
+		c = p->mcache;
+		if(c==nil)
+			continue;
+		runtime·purgecachedstats(c);
 		for(i=0; i<nelem(c->local_by_size); i++) {
 			mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc;
 			c->local_by_size[i].nmalloc = 0;
@@ -1819,12 +1815,11 @@ gc(struct gc_args *args)
 	runtime·parfordo(work.sweepfor);
 	t3 = runtime·nanotime();
 
-	stealcache();
-	cachestats(&stats);
-
 	if(work.nproc > 1)
 		runtime·notesleep(&work.alldone);
 
+	cachestats(&stats);
+
 	stats.nprocyield += work.sweepfor->nprocyield;
 	stats.nosyield += work.sweepfor->nosyield;
 	stats.nsleep += work.sweepfor->nsleep;

src/pkg/runtime/runtime.h

@@ -118,10 +118,19 @@ enum
 	Grunning,
 	Gsyscall,
 	Gwaiting,
-	Gmoribund,
+	Gmoribund_unused,  // currently unused, but hardcoded in gdb scripts
 	Gdead,
 };
 enum
+{
+	// P status
+	Pidle,
+	Prunning,
+	Psyscall,
+	Pgcstop,
+	Pdead,
+};
+enum
 {
 	true	= 1,
 	false	= 0,
@@ -214,6 +223,7 @@ struct	G
 	Gobuf	sched;
 	uintptr	gcstack;		// if status==Gsyscall, gcstack = stackbase to use during gc
 	uintptr	gcsp;		// if status==Gsyscall, gcsp = sched.sp to use during gc
+	byte*	gcpc;		// if status==Gsyscall, gcpc = sched.pc to use during gc
 	uintptr	gcguard;		// if status==Gsyscall, gcguard = stackguard to use during gc
 	uintptr	stack0;
 	FuncVal*	fnstart;		// initial function
@@ -224,13 +234,11 @@ struct	G
 	uint32	selgen;		// valid sudog pointer
 	int8*	waitreason;	// if status==Gwaiting
 	G*	schedlink;
-	bool	readyonstop;
 	bool	ispanic;
 	bool	issystem;
 	int8	raceignore; // ignore race detection events
 	M*	m;		// for debuggers, but offset not hard-coded
 	M*	lockedm;
-	M*	idlem;
 	int32	sig;
 	int32	writenbuf;
 	byte*	writebuf;
@@ -259,22 +267,24 @@ struct	M
 	G*	gsignal;	// signal-handling G
 	uint32	tls[8];		// thread-local storage (for 386 extern register)
 	G*	curg;		// current running goroutine
+	P*	p;		// attached P for executing Go code (nil if not executing Go code)
+	P*	nextp;
 	int32	id;
 	int32	mallocing;
 	int32	throwing;
 	int32	gcing;
 	int32	locks;
 	int32	nomemprof;
-	int32	waitnextg;
 	int32	dying;
 	int32	profilehz;
 	int32	helpgc;
+	bool	blockingsyscall;
+	bool	spinning;
 	uint32	fastrand;
 	uint64	ncgocall;	// number of cgo calls in total
 	int32	ncgo;		// number of cgo calls currently in progress
 	CgoMal*	cgomal;
-	Note	havenextg;
-	G*	nextg;
+	Note	park;
 	M*	alllink;	// on allm
 	M*	schedlink;
 	uint32	machport;	// Return address for Mach IPC (OS X)
@@ -284,7 +294,6 @@ struct	M
 	uint32	stackcachecnt;
 	void*	stackcache[StackCacheSize];
 	G*	lockedg;
-	G*	idleg;
 	uintptr	createstack[32];	// Stack that created this thread.
 	uint32	freglo[16];	// D[i] lsb and F[i]
 	uint32	freghi[16];	// D[i] msb and F[i+16]
@@ -298,6 +307,8 @@ struct	M
 	bool	racecall;
 	bool	needextram;
 	void*	racepc;
+	void	(*waitunlockf)(Lock*);
+	Lock*	waitlock;
 	uint32	moreframesize_minalloc;
 
 	uintptr	settype_buf[1024];
@@ -317,7 +328,11 @@ struct P
 {
 	Lock;
 
+	uint32	status;  // one of Pidle/Prunning/...
 	P*	link;
+	uint32	tick;   // incremented on every scheduler or system call
+	M*	m;	// back-link to associated M (nil if idle)
+	MCache*	mcache;
 
 	// Queue of runnable goroutines.
 	G**	runq;
@@ -608,6 +623,7 @@ extern	uintptr runtime·zerobase;
 extern	G*	runtime·allg;
 extern	G*	runtime·lastg;
 extern	M*	runtime·allm;
+extern	P**	runtime·allp;
 extern	int32	runtime·gomaxprocs;
 extern	bool	runtime·singleproc;
 extern	uint32	runtime·panicking;