cmd/compile: add go:notinheap type pragma

This adds a //go:notinheap pragma for declarations of types that must
not be heap allocated. We ensure these rules by disallowing new(T),
make([]T), append([]T), or implicit allocation of T, by disallowing
conversions to notinheap types, and by propagating notinheap to any
struct or array that contains notinheap elements.

The utility of this pragma is that we can eliminate write barriers for
writes to pointers to go:notinheap types, since the write barrier is
guaranteed to be a no-op. This will let us mark several scheduler and
memory allocator structures as go:notinheap, which will let us
disallow write barriers in the scheduler and memory allocator much
more thoroughly and also eliminate some problematic hybrid write
barriers.

This also makes go:nowritebarrierrec and go:yeswritebarrierrec much
more powerful. Currently we use go:nowritebarrier all over the place,
but it's almost never what you actually want: when write barriers are
illegal, they're typically illegal for a whole dynamic scope. Partly
this is because go:nowritebarrier has been around longer, but it's
also because go:nowritebarrierrec couldn't be used in situations that
had no-op write barriers or where some nested scope did allow write
barriers. go:notinheap eliminates many no-op write barriers and
go:yeswritebarrierrec makes it possible to opt back in to write
barriers, so these two changes will let us use go:nowritebarrierrec
far more liberally.

This updates #13386, which is about controlling pointers from non-GC'd
memory to GC'd memory. That would require some additional pragma (or
pragmas), but could build on this pragma.

Change-Id: I6314f8f4181535dd166887c9ec239977b54940bd
Reviewed-on: https://go-review.googlesource.com/30939Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>

src/cmd/compile/internal/gc/lex.go

@@ -64,6 +64,7 @@ func plan9quote(s string) string {
 type Pragma syntax.Pragma
 
 const (
+	// Func pragmas.
 	Nointerface    Pragma = 1 << iota
 	Noescape              // func parameters don't escape
 	Norace                // func must not have race detector annotations
@@ -72,13 +73,15 @@ const (
 	CgoUnsafeArgs         // treat a pointer to one arg as a pointer to them all
 	UintptrEscapes        // pointers converted to uintptr escape
 
-	// Runtime-only pragmas.
+	// Runtime-only func pragmas.
 	// See ../../../../runtime/README.md for detailed descriptions.
-
 	Systemstack        // func must run on system stack
 	Nowritebarrier     // emit compiler error instead of write barrier
 	Nowritebarrierrec  // error on write barrier in this or recursive callees
 	Yeswritebarrierrec // cancels Nowritebarrierrec in this function and callees
+
+	// Runtime-only type pragmas
+	NotInHeap // values of this type must not be heap allocated
 )
 
 func pragmaValue(verb string) Pragma {
@@ -130,6 +133,8 @@ func pragmaValue(verb string) Pragma {
 		// in the argument list.
 		// Used in syscall/dll_windows.go.
 		return UintptrEscapes
+	case "go:notinheap":
+		return NotInHeap
 	}
 	return 0
 }

src/cmd/compile/internal/gc/noder.go

@@ -188,6 +188,7 @@ func (p *noder) constDecl(decl *syntax.ConstDecl) []*Node {
 
 func (p *noder) typeDecl(decl *syntax.TypeDecl) *Node {
 	name := typedcl0(p.name(decl.Name))
+	name.Name.Param.Pragma = Pragma(decl.Pragma)
 
 	var typ *Node
 	if decl.Type != nil {

src/cmd/compile/internal/gc/parser.go

@@ -479,6 +479,7 @@ func (p *parser) typedcl() []*Node {
 	}
 
 	name := typedcl0(p.sym())
+	name.Name.Param.Pragma = p.pragma
 
 	typ := p.try_ntype()
 	// handle case where type is missing

src/cmd/compile/internal/gc/subr.go

@@ -863,6 +863,16 @@ func convertop(src *Type, dst *Type, why *string) Op {
 		return 0
 	}
 
+	// Conversions from regular to go:notinheap are not allowed
+	// (unless it's unsafe.Pointer). This is a runtime-specific
+	// rule.
+	if src.IsPtr() && dst.IsPtr() && dst.Elem().NotInHeap && !src.Elem().NotInHeap {
+		if why != nil {
+			*why = fmt.Sprintf(":\n\t%v is go:notinheap, but %v is not", dst.Elem(), src.Elem())
+		}
+		return 0
+	}
+
 	// 1. src can be assigned to dst.
 	op := assignop(src, dst, why)
 	if op != 0 {

src/cmd/compile/internal/gc/syntax.go

@@ -266,6 +266,11 @@ type Param struct {
 	// and x.Innermost/Outer means x.Name.Param.Innermost/Outer.
 	Innermost *Node
 	Outer     *Node
+
+	// OTYPE pragmas
+	//
+	// TODO: Should Func pragmas also be stored on the Name?
+	Pragma Pragma
 }
 
 // Func holds Node fields used only with function-like nodes.

src/cmd/compile/internal/gc/type.go

@@ -160,6 +160,7 @@ type Type struct {
 	Deferwidth bool
 	Broke      bool  // broken type definition.
 	Align      uint8 // the required alignment of this type, in bytes
+	NotInHeap  bool  // type cannot be heap allocated
 }
 
 // MapType contains Type fields specific to maps.
@@ -414,6 +415,7 @@ func typArray(elem *Type, bound int64) *Type {
 	}
 	t := typ(TARRAY)
 	t.Extra = &ArrayType{Elem: elem, Bound: bound}
+	t.NotInHeap = elem.NotInHeap
 	return t
 }
 
@@ -436,6 +438,7 @@ func typSlice(elem *Type) *Type {
 func typDDDArray(elem *Type) *Type {
 	t := typ(TARRAY)
 	t.Extra = &ArrayType{Elem: elem, Bound: -1}
+	t.NotInHeap = elem.NotInHeap
 	return t
 }
 
@@ -822,6 +825,17 @@ func (t *Type) FieldSlice() []*Field {
 
 // SetFields sets struct/interface type t's fields/methods to fields.
 func (t *Type) SetFields(fields []*Field) {
+	for _, f := range fields {
+		// If type T contains a field F with a go:notinheap
+		// type, then T must also be go:notinheap. Otherwise,
+		// you could heap allocate T and then get a pointer F,
+		// which would be a heap pointer to a go:notinheap
+		// type.
+		if f.Type != nil && f.Type.NotInHeap {
+			t.NotInHeap = true
+			break
+		}
+	}
 	t.Fields().Set(fields)
 }
 

src/cmd/compile/internal/gc/typecheck.go

@@ -403,6 +403,12 @@ OpSwitch:
 			n.Type = nil
 			return n
 		}
+		if l.Type.NotInHeap {
+			yyerror("go:notinheap map key not allowed")
+		}
+		if r.Type.NotInHeap {
+			yyerror("go:notinheap map value not allowed")
+		}
 		n.Op = OTYPE
 		n.Type = typMap(l.Type, r.Type)
 
@@ -428,6 +434,9 @@ OpSwitch:
 			n.Type = nil
 			return n
 		}
+		if l.Type.NotInHeap {
+			yyerror("chan of go:notinheap type not allowed")
+		}
 		t := typChan(l.Type, ChanDir(n.Etype)) // TODO(marvin): Fix Node.EType type union.
 		n.Op = OTYPE
 		n.Type = t
@@ -2087,6 +2096,12 @@ OpSwitch:
 		ok |= Etop
 		n.Left = typecheck(n.Left, Etype)
 		checkwidth(n.Left.Type)
+		if n.Left.Type != nil && n.Left.Type.NotInHeap && n.Left.Name.Param.Pragma&NotInHeap == 0 {
+			// The type contains go:notinheap types, so it
+			// must be marked as such (alternatively, we
+			// could silently propagate go:notinheap).
+			yyerror("type %v must be go:notinheap", n.Left.Type)
+		}
 		break OpSwitch
 	}
 
@@ -3516,6 +3531,11 @@ func copytype(n *Node, t *Type) {
 	t.ptrTo = ptrTo
 	t.sliceOf = sliceOf
 
+	// Propagate go:notinheap pragma from the Name to the Type.
+	if n.Name != nil && n.Name.Param != nil && n.Name.Param.Pragma&NotInHeap != 0 {
+		t.NotInHeap = true
+	}
+
 	// Update nodes waiting on this type.
 	for _, n := range l {
 		copytype(n, t)

src/cmd/compile/internal/gc/walk.go

@@ -773,6 +773,9 @@ opswitch:
 		case OAPPEND:
 			// x = append(...)
 			r := n.Right
+			if r.Type.Elem().NotInHeap {
+				yyerror("%v is go:notinheap; heap allocation disallowed", r.Type.Elem())
+			}
 			if r.Isddd {
 				r = appendslice(r, init) // also works for append(slice, string).
 			} else {
@@ -1546,6 +1549,10 @@ opswitch:
 			// When len and cap can fit into int, use makeslice instead of
 			// makeslice64, which is faster and shorter on 32 bit platforms.
 
+			if t.Elem().NotInHeap {
+				yyerror("%v is go:notinheap; heap allocation disallowed", t.Elem())
+			}
+
 			len, cap := l, r
 
 			fnname := "makeslice64"
@@ -2146,6 +2153,9 @@ func walkprint(nn *Node, init *Nodes) *Node {
 }
 
 func callnew(t *Type) *Node {
+	if t.NotInHeap {
+		yyerror("%v is go:notinheap; heap allocation disallowed", t)
+	}
 	dowidth(t)
 	fn := syslook("newobject")
 	fn = substArgTypes(fn, t)
@@ -2217,6 +2227,12 @@ func needwritebarrier(l *Node, r *Node) bool {
 		return false
 	}
 
+	// No write barrier if this is a pointer to a go:notinheap
+	// type, since the write barrier's inheap(ptr) check will fail.
+	if l.Type.IsPtr() && l.Type.Elem().NotInHeap {
+		return false
+	}
+
 	// Ignore no-op conversions when making decision.
 	// Ensures that xp = unsafe.Pointer(&x) is treated
 	// the same as xp = &x.

src/cmd/compile/internal/syntax/nodes.go

@@ -91,6 +91,7 @@ type (
 		Type   Expr
 		Alias  bool
 		Group  *Group // nil means not part of a group
+		Pragma Pragma
 		decl
 	}
 

src/cmd/compile/internal/syntax/parser.go

@@ -381,6 +381,7 @@ func (p *parser) typeDecl(group *Group) Decl {
 		p.advance(_Semi, _Rparen)
 	}
 	d.Group = group
+	d.Pragma = p.pragma
 
 	return d
 }

src/cmd/compile/internal/syntax/syntax.go

@@ -12,8 +12,8 @@ import (
 
 type Mode uint
 
-// A Pragma value is a set of flags that augment a function
-// declaration. Callers may assign meaning to the flags as
+// A Pragma value is a set of flags that augment a function or
+// type declaration. Callers may assign meaning to the flags as
 // appropriate.
 type Pragma uint16
 

src/runtime/HACKING.md

@@ -47,3 +47,36 @@ functions that release the P or may run without a P and
 `go:yeswritebarrierrec` is used when code re-acquires an active P.
 Since these are function-level annotations, code that releases or
 acquires a P may need to be split across two functions.
+
+go:notinheap
+------------
+
+`go:notinheap` applies to type declarations. It indicates that a type
+must never be heap allocated. Specifically, pointers to this type must
+always fail the `runtime.inheap` check. The type may be used for
+global variables, for stack variables, or for objects in unmanaged
+memory (e.g., allocated with `sysAlloc`, `persistentalloc`, or
+`fixalloc`). Specifically:
+
+1. `new(T)`, `make([]T)`, `append([]T, ...)` and implicit heap
+   allocation of T are disallowed. (Though implicit allocations are
+   disallowed in the runtime anyway.)
+
+2. A pointer to a regular type (other than `unsafe.Pointer`) cannot be
+   converted to a pointer to a `go:notinheap` type, even if they have
+   the same underlying type.
+
+3. Any type that contains a `go:notinheap` type is itself
+   `go:notinheap`. Structs and arrays are `go:notinheap` if their
+   elements are. Maps and channels of `go:notinheap` types are
+   disallowed. To keep things explicit, any type declaration where the
+   type is implicitly `go:notinheap` must be explicitly marked
+   `go:notinheap` as well.
+
+4. Write barriers on pointers to `go:notinheap` types can be omitted.
+
+The last point is the real benefit of `go:notinheap`. The runtime uses
+it for low-level internal structures to avoid memory barriers in the
+scheduler and the memory allocator where they are illegal or simply
+inefficient. This mechanism is reasonably safe and does not compromise
+the readability of the runtime.

test/notinheap.go

+// errorcheck -+
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Test type-checking errors for go:notinheap.
+
+package p
+
+//go:notinheap
+type nih struct{}
+
+// Types embedding notinheap types must be notinheap.
+
+type embed1 struct {
+	x nih
+} // ERROR "must be go:notinheap"
+
+type embed2 [1]nih // ERROR "must be go:notinheap"
+
+type embed3 struct {
+	x [1]nih
+} // ERROR "must be go:notinheap"
+
+type embed4 map[nih]int // ERROR "go:notinheap map key not allowed"
+
+type embed5 map[int]nih // ERROR "go:notinheap map value not allowed"
+
+type emebd6 chan nih // ERROR "chan of go:notinheap type not allowed"
+
+type okay1 *nih
+
+type okay2 []nih
+
+type okay3 func(x nih) nih
+
+type okay4 interface {
+	f(x nih) nih
+}
+
+// Type conversions don't let you sneak past notinheap.
+
+type t1 struct{ x int }
+
+//go:notinheap
+type t2 t1
+
+var sink interface{}
+
+func i() {
+	sink = new(t1)                     // no error
+	sink = (*t2)(new(t1))              // ERROR "cannot convert(.|\n)*t2 is go:notinheap"
+	sink = (*t2)(new(struct{ x int })) // ERROR "cannot convert(.|\n)*t2 is go:notinheap"
+}

test/notinheap2.go

+// errorcheck -+
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Test walk errors for go:notinheap.
+
+package p
+
+//go:notinheap
+type nih struct {
+	next *nih
+}
+
+// Globals and stack variables are okay.
+
+var x nih
+
+func f() {
+	var y nih
+	x = y
+}
+
+// Heap allocation is not okay.
+
+var y *nih
+var z []nih
+
+func g() {
+	y = new(nih)       // ERROR "heap allocation disallowed"
+	z = make([]nih, 1) // ERROR "heap allocation disallowed"
+	z = append(z, x)   // ERROR "heap allocation disallowed"
+}
+
+// Writes don't produce write barriers.
+
+var p *nih
+
+//go:nowritebarrier
+func h() {
+	y.next = p.next
+}