undo CL 104200047 / 318b04f28372

Breaks windows and race detector.
TBR=rsc

««« original CL description
runtime: stack allocator, separate from mallocgc

In order to move malloc to Go, we need to have a
separate stack allocator.  If we run out of stack
during malloc, malloc will not be available
to allocate a new stack.

Stacks are the last remaining FlagNoGC objects in the
GC heap.  Once they are out, we can get rid of the
distinction between the allocated/blockboundary bits.
(This will be in a separate change.)

Fixes #7468
Fixes #7424

LGTM=rsc, dvyukov
R=golang-codereviews, dvyukov, khr, dave, rsc
CC=golang-codereviews
https://golang.org/cl/104200047
»»»

TBR=rsc
CC=golang-codereviews
https://golang.org/cl/101570044

src/pkg/runtime/malloc.h

@@ -116,12 +116,6 @@ enum
 	MaxMHeapList = 1<<(20 - PageShift),	// Maximum page length for fixed-size list in MHeap.
 	HeapAllocChunk = 1<<20,		// Chunk size for heap growth
 
-	// Per-P, per order stack segment cache size.
-	StackCacheSize = 32*1024,
-	// Number of orders that get caching.  Order 0 is StackMin
-	// and each successive order is twice as large.
-	NumStackOrders = 3,
-
 	// Number of bits in page to span calculations (4k pages).
 	// On Windows 64-bit we limit the arena to 32GB or 35 bits (see below for reason).
 	// On other 64-bit platforms, we limit the arena to 128GB, or 37 bits.
@@ -253,8 +247,8 @@ struct MStats
 
 	// Statistics about allocation of low-level fixed-size structures.
 	// Protected by FixAlloc locks.
-	uint64	stacks_inuse;	// this number is included in heap_inuse above
-	uint64	stacks_sys;	// always 0 in mstats
+	uint64	stacks_inuse;	// bootstrap stacks
+	uint64	stacks_sys;
 	uint64	mspan_inuse;	// MSpan structures
 	uint64	mspan_sys;
 	uint64	mcache_inuse;	// MCache structures
@@ -311,13 +305,6 @@ struct MCacheList
 	uint32 nlist;
 };
 
-typedef struct StackFreeList StackFreeList;
-struct StackFreeList
-{
-	MLink *list;  // linked list of free stacks
-	uintptr size; // total size of stacks in list
-};
-
 // Per-thread (in Go, per-P) cache for small objects.
 // No locking needed because it is per-thread (per-P).
 struct MCache
@@ -333,9 +320,6 @@ struct MCache
 	// The rest is not accessed on every malloc.
 	MSpan*	alloc[NumSizeClasses];	// spans to allocate from
 	MCacheList free[NumSizeClasses];// lists of explicitly freed objects
-
-	StackFreeList stackcache[NumStackOrders];
-
 	// Local allocator stats, flushed during GC.
 	uintptr local_nlookup;		// number of pointer lookups
 	uintptr local_largefree;	// bytes freed for large objects (>MaxSmallSize)
@@ -346,7 +330,6 @@ struct MCache
 MSpan*	runtime·MCache_Refill(MCache *c, int32 sizeclass);
 void	runtime·MCache_Free(MCache *c, MLink *p, int32 sizeclass, uintptr size);
 void	runtime·MCache_ReleaseAll(MCache *c);
-void	runtime·stackcache_clear(MCache *c);
 
 // MTypes describes the types of blocks allocated within a span.
 // The compression field describes the layout of the data.
@@ -426,8 +409,7 @@ struct SpecialProfile
 // An MSpan is a run of pages.
 enum
 {
-	MSpanInUse = 0, // allocated for garbage collected heap
-	MSpanStack,     // allocated for use by stack allocator
+	MSpanInUse = 0,
 	MSpanFree,
 	MSpanListHead,
 	MSpanDead,
@@ -543,9 +525,7 @@ extern MHeap runtime·mheap;
 
 void	runtime·MHeap_Init(MHeap *h);
 MSpan*	runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large, bool needzero);
-MSpan*	runtime·MHeap_AllocStack(MHeap *h, uintptr npage);
 void	runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct);
-void	runtime·MHeap_FreeStack(MHeap *h, MSpan *s);
 MSpan*	runtime·MHeap_Lookup(MHeap *h, void *v);
 MSpan*	runtime·MHeap_LookupMaybe(MHeap *h, void *v);
 void	runtime·MGetSizeClassInfo(int32 sizeclass, uintptr *size, int32 *npages, int32 *nobj);
@@ -553,6 +533,7 @@ void*	runtime·MHeap_SysAlloc(MHeap *h, uintptr n);
 void	runtime·MHeap_MapBits(MHeap *h);
 void	runtime·MHeap_MapSpans(MHeap *h);
 void	runtime·MHeap_Scavenger(void);
+void	runtime·MHeap_SplitSpan(MHeap *h, MSpan *s);
 
 void*	runtime·mallocgc(uintptr size, uintptr typ, uint32 flag);
 void*	runtime·persistentalloc(uintptr size, uintptr align, uint64 *stat);

src/pkg/runtime/mcache.c

@@ -43,7 +43,6 @@ void
 runtime·freemcache(MCache *c)
 {
 	runtime·MCache_ReleaseAll(c);
-	runtime·stackcache_clear(c);
 	runtime·lock(&runtime·mheap);
 	runtime·purgecachedstats(c);
 	runtime·FixAlloc_Free(&runtime·mheap.cachealloc, c);

src/pkg/runtime/mcentral.c

@@ -263,8 +263,6 @@ MCentral_Grow(MCentral *c)
 	runtime·unlock(c);
 	runtime·MGetSizeClassInfo(c->sizeclass, &size, &npages, &n);
 	s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1);
-	if(s->next != nil || s->prev != nil)
-		runtime·throw("internal error: MSpan should not be in a list");
 	if(s == nil) {
 		// TODO(rsc): Log out of memory
 		runtime·lock(c);

src/pkg/runtime/mem.go

@@ -30,7 +30,7 @@ type MemStats struct {
 	// Low-level fixed-size structure allocator statistics.
 	//	Inuse is bytes used now.
 	//	Sys is bytes obtained from system.
-	StackInuse  uint64 // bytes used by stack allocator
+	StackInuse  uint64 // bootstrap stacks
 	StackSys    uint64
 	MSpanInuse  uint64 // mspan structures
 	MSpanSys    uint64

src/pkg/runtime/mgc0.c

@@ -1252,12 +1252,12 @@ markroot(ParFor *desc, uint32 i)
 			SpecialFinalizer *spf;
 
 			s = allspans[spanidx];
-			if(s->state != MSpanInUse)
-				continue;
 			if(s->sweepgen != sg) {
 				runtime·printf("sweep %d %d\n", s->sweepgen, sg);
 				runtime·throw("gc: unswept span");
 			}
+			if(s->state != MSpanInUse)
+				continue;
 			// The garbage collector ignores type pointers stored in MSpan.types:
 			//  - Compiler-generated types are stored outside of heap.
 			//  - The reflect package has runtime-generated types cached in its data structures.
@@ -2124,7 +2124,6 @@ flushallmcaches(void)
 		if(c==nil)
 			continue;
 		runtime·MCache_ReleaseAll(c);
-		runtime·stackcache_clear(c);
 	}
 }
 
@@ -2134,12 +2133,14 @@ runtime·updatememstats(GCStats *stats)
 	M *mp;
 	MSpan *s;
 	int32 i;
-	uint64 smallfree;
+	uint64 stacks_inuse, smallfree;
 	uint64 *src, *dst;
 
 	if(stats)
 		runtime·memclr((byte*)stats, sizeof(*stats));
+	stacks_inuse = 0;
 	for(mp=runtime·allm; mp; mp=mp->alllink) {
+		stacks_inuse += mp->stackinuse*FixedStack;
 		if(stats) {
 			src = (uint64*)&mp->gcstats;
 			dst = (uint64*)stats;
@@ -2148,6 +2149,7 @@ runtime·updatememstats(GCStats *stats)
 			runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats));
 		}
 	}
+	mstats.stacks_inuse = stacks_inuse;
 	mstats.mcache_inuse = runtime·mheap.cachealloc.inuse;
 	mstats.mspan_inuse = runtime·mheap.spanalloc.inuse;
 	mstats.sys = mstats.heap_sys + mstats.stacks_sys + mstats.mspan_sys +
@@ -2507,12 +2509,6 @@ runtime·ReadMemStats(MStats *stats)
 	// Size of the trailing by_size array differs between Go and C,
 	// NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
 	runtime·memcopy(runtime·sizeof_C_MStats, stats, &mstats);
-
-	// Stack numbers are part of the heap numbers, separate those out for user consumption
-	stats->stacks_sys = stats->stacks_inuse;
-	stats->heap_inuse -= stats->stacks_inuse;
-	stats->heap_sys -= stats->stacks_inuse;
-	
 	g->m->gcing = 0;
 	g->m->locks++;
 	runtime·semrelease(&runtime·worldsema);

src/pkg/runtime/proc.c

@@ -152,7 +152,6 @@ runtime·schedinit(void)
 	runtime·precisestack = true; // haveexperiment("precisestack");
 
 	runtime·symtabinit();
-	runtime·stackinit();
 	runtime·mallocinit();
 	mcommoninit(g->m);
 	
@@ -1927,7 +1926,7 @@ gfput(P *p, G *gp)
 		runtime·throw("gfput: bad stacksize");
 	}
 	top = (Stktop*)gp->stackbase;
-	if(stksize != FixedStack) {
+	if(top->malloced) {
 		// non-standard stack size - free it.
 		runtime·stackfree(gp, (void*)gp->stack0, top);
 		gp->stack0 = 0;

src/pkg/runtime/runtime.h

@@ -146,6 +146,13 @@ enum
 {
 	PtrSize = sizeof(void*),
 };
+enum
+{
+	// Per-M stack segment cache size.
+	StackCacheSize = 32,
+	// Global <-> per-M stack segment cache transfer batch size.
+	StackCacheBatch = 16,
+};
 /*
  * structures
  */
@@ -319,6 +326,10 @@ struct	M
 	M*	schedlink;
 	uint32	machport;	// Return address for Mach IPC (OS X)
 	MCache*	mcache;
+	int32	stackinuse;
+	uint32	stackcachepos;
+	uint32	stackcachecnt;
+	void*	stackcache[StackCacheSize];
 	G*	lockedg;
 	uintptr	createstack[32];// Stack that created this thread.
 	uint32	freglo[16];	// D[i] lsb and F[i]
@@ -335,8 +346,6 @@ struct	M
 	bool	(*waitunlockf)(G*, void*);
 	void*	waitlock;
 	uintptr	forkstackguard;
-	uintptr scalararg[4];	// scalar argument/return for mcall
-	void*   ptrarg[4];	// pointer argument/return for mcall
 #ifdef GOOS_windows
 	void*	thread;		// thread handle
 	// these are here because they are too large to be on the stack
@@ -419,6 +428,7 @@ struct	Stktop
 
 	uint8*	argp;	// pointer to arguments in old frame
 	bool	panic;	// is this frame the top of a panic?
+	bool	malloced;
 };
 struct	SigTab
 {
@@ -856,7 +866,6 @@ int32	runtime·funcarglen(Func*, uintptr);
 int32	runtime·funcspdelta(Func*, uintptr);
 int8*	runtime·funcname(Func*);
 int32	runtime·pcdatavalue(Func*, int32, uintptr);
-void	runtime·stackinit(void);
 void*	runtime·stackalloc(G*, uint32);
 void	runtime·stackfree(G*, void*, Stktop*);
 void	runtime·shrinkstack(G*);

src/pkg/runtime/stack_test.go

@@ -281,52 +281,3 @@ func TestDeferPtrs(t *testing.T) {
 	defer set(&y, 42)
 	growStack()
 }
-
-// use about n KB of stack
-func useStack(n int) {
-	if n == 0 {
-		return
-	}
-	var b [1024]byte // makes frame about 1KB
-	useStack(n - 1 + int(b[99]))
-}
-
-func growing(c chan int, done chan struct{}) {
-	for n := range c {
-		useStack(n)
-		done <- struct{}{}
-	}
-	done <- struct{}{}
-}
-
-func TestStackCache(t *testing.T) {
-	// Allocate a bunch of goroutines and grow their stacks.
-	// Repeat a few times to test the stack cache.
-	const (
-		R = 4
-		G = 200
-		S = 5
-	)
-	for i := 0; i < R; i++ {
-		var reqchans [G]chan int
-		done := make(chan struct{})
-		for j := 0; j < G; j++ {
-			reqchans[j] = make(chan int)
-			go growing(reqchans[j], done)
-		}
-		for s := 0; s < S; s++ {
-			for j := 0; j < G; j++ {
-				reqchans[j] <- 1 << uint(s)
-			}
-			for j := 0; j < G; j++ {
-				<-done
-			}
-		}
-		for j := 0; j < G; j++ {
-			close(reqchans[j])
-		}
-		for j := 0; j < G; j++ {
-			<-done
-		}
-	}
-}