cmd/compile: print recursive types correctly

Change the type printer to take a map of types that we're currently
printing. When we happen upon a type that we're already in the middle
of printing, print a reference to it instead.

A reference to another type is built using the offset of the first
byte of that type's string representation in the result. To facilitate
that computation (and it's probably more efficient, regardless), we
print the type to a buffer as we go, and build the string at the end.

It would be nice to use string.Builder instead of bytes.Buffer, but
string.Builder wasn't around in Go 1.4, and we'd like to bootstrap
from that version.

Fixes #29312

Change-Id: I49d788c1fa20f770df7b2bae3b9979d990d54803
Reviewed-on: https://go-review.googlesource.com/c/go/+/214239Reviewed-by: Robert Griesemer <gri@golang.org>

src/cmd/compile/internal/gc/main.go

@@ -512,8 +512,8 @@ func Main(archInit func(*Arch)) {
 	types.Sconv = func(s *types.Sym, flag, mode int) string {
 		return sconv(s, FmtFlag(flag), fmtMode(mode))
 	}
-	types.Tconv = func(t *types.Type, flag, mode, depth int) string {
-		return tconv(t, FmtFlag(flag), fmtMode(mode), depth)
+	types.Tconv = func(t *types.Type, flag, mode int) string {
+		return tconv(t, FmtFlag(flag), fmtMode(mode))
 	}
 	types.FormatSym = func(sym *types.Sym, s fmt.State, verb rune, mode int) {
 		symFormat(sym, s, verb, fmtMode(mode))

src/cmd/compile/internal/ssa/export_test.go

@@ -153,7 +153,7 @@ func init() {
 	// TODO(josharian): move universe initialization to the types package,
 	// so this test setup can share it.
 
-	types.Tconv = func(t *types.Type, flag, mode, depth int) string {
+	types.Tconv = func(t *types.Type, flag, mode int) string {
 		return t.Etype.String()
 	}
 	types.Sconv = func(s *types.Sym, flag, mode int) string {

src/cmd/compile/internal/types/utils.go

@@ -19,7 +19,7 @@ var (
 	Dowidth     func(*Type)
 	Fatalf      func(string, ...interface{})
 	Sconv       func(*Sym, int, int) string       // orig: func sconv(s *Sym, flag FmtFlag, mode fmtMode) string
-	Tconv       func(*Type, int, int, int) string // orig: func tconv(t *Type, flag FmtFlag, mode fmtMode, depth int) string
+	Tconv       func(*Type, int, int) string      // orig: func tconv(t *Type, flag FmtFlag, mode fmtMode) string
 	FormatSym   func(*Sym, fmt.State, rune, int)  // orig: func symFormat(sym *Sym, s fmt.State, verb rune, mode fmtMode)
 	FormatType  func(*Type, fmt.State, rune, int) // orig: func typeFormat(t *Type, s fmt.State, verb rune, mode fmtMode)
 	TypeLinkSym func(*Type) *obj.LSym
@@ -39,25 +39,23 @@ func (sym *Sym) Format(s fmt.State, verb rune) {
 }
 
 func (t *Type) String() string {
-	// This is an external entry point, so we pass depth 0 to tconv.
 	// The implementation of tconv (including typefmt and fldconv)
-	// must take care not to use a type in a formatting string
-	// to avoid resetting the recursion counter.
-	return Tconv(t, 0, FErr, 0)
+	// must handle recursive types correctly.
+	return Tconv(t, 0, FErr)
 }
 
 // ShortString generates a short description of t.
 // It is used in autogenerated method names, reflection,
 // and itab names.
 func (t *Type) ShortString() string {
-	return Tconv(t, FmtLeft, FErr, 0)
+	return Tconv(t, FmtLeft, FErr)
 }
 
 // LongString generates a complete description of t.
 // It is useful for reflection,
 // or when a unique fingerprint or hash of a type is required.
 func (t *Type) LongString() string {
-	return Tconv(t, FmtLeft|FmtUnsigned, FErr, 0)
+	return Tconv(t, FmtLeft|FmtUnsigned, FErr)
 }
 
 func (t *Type) Format(s fmt.State, verb rune) {