cmd/compile: rework mkbuiltin.go to generate code

Generating binary export data requires a working Go compiler. Even
trickier to change the export data format itself requires a careful
bootstrapping procedure.

Instead, simply generate normal Go code that lets us directly
construct the builtin runtime declarations.

Passes toolstash -cmp.

Fixes #17508.

Change-Id: I4f6078a3c7507ba40072580695d57c87a5604baf
Reviewed-on: https://go-review.googlesource.com/31493
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

src/cmd/compile/internal/gc/dcl.go

@@ -372,6 +372,14 @@ func typenod(t *Type) *Node {
 	return t.nod
 }
 
+func anonfield(typ *Type) *Node {
+	return nod(ODCLFIELD, nil, typenod(typ))
+}
+
+func namedfield(s string, typ *Type) *Node {
+	return nod(ODCLFIELD, newname(lookup(s)), typenod(typ))
+}
+
 // oldname returns the Node that declares symbol s in the current scope.
 // If no such Node currently exists, an ONONAME Node is returned instead.
 func oldname(s *Sym) *Node {

src/cmd/compile/internal/gc/main.go

@@ -673,9 +673,9 @@ func findpkg(name string) (file string, ok bool) {
 	return "", false
 }
 
-// loadsys loads the definitions for the low-level runtime and unsafe functions,
+// loadsys loads the definitions for the low-level runtime functions,
 // so that the compiler can generate calls to them,
-// but does not make the names "runtime" or "unsafe" visible as packages.
+// but does not make them visible to user code.
 func loadsys() {
 	if Debug['A'] != 0 {
 		return
@@ -685,7 +685,28 @@ func loadsys() {
 	iota_ = -1000000
 
 	importpkg = Runtimepkg
-	Import(bufio.NewReader(strings.NewReader(runtimeimport)))
+	typecheckok = true
+	defercheckwidth()
+
+	typs := runtimeTypes()
+	for _, d := range runtimeDecls {
+		sym := Pkglookup(d.name, importpkg)
+		typ := typs[d.typ]
+		switch d.tag {
+		case funcTag:
+			importsym(sym, ONAME)
+			n := newfuncname(sym)
+			n.Type = typ
+			declare(n, PFUNC)
+		case varTag:
+			importvar(sym, typ)
+		default:
+			Fatalf("unhandled declaration tag %v", d.tag)
+		}
+	}
+
+	typecheckok = false
+	resumecheckwidth()
 	importpkg = nil
 }
 

src/cmd/compile/internal/gc/mkbuiltin.go

-// Copyright 2009 The Go Authors. All rights reserved.
+// Copyright 2016 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // +build ignore
 
 // Generate builtin.go from builtin/runtime.go.
-// Run this after changing builtin/runtime.go
-// or after changing the export metadata format in the compiler.
-// Either way, you need to have a working compiler binary first.
-// See bexport.go for how to make an export metadata format change.
+
 package main
 
 import (
 	"bytes"
 	"flag"
 	"fmt"
+	"go/ast"
+	"go/format"
+	"go/parser"
+	"go/token"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
-	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
 )
 
 var stdout = flag.Bool("stdout", false, "write to stdout instead of builtin.go")
@@ -29,65 +32,181 @@ func main() {
 
 	var b bytes.Buffer
 	fmt.Fprintln(&b, "// AUTO-GENERATED by mkbuiltin.go; DO NOT EDIT")
-	fmt.Fprintln(&b, "")
+	fmt.Fprintln(&b)
 	fmt.Fprintln(&b, "package gc")
 
 	mkbuiltin(&b, "runtime")
 
-	var err error
+	out, err := format.Source(b.Bytes())
+	if err != nil {
+		log.Fatal(err)
+	}
 	if *stdout {
-		_, err = os.Stdout.Write(b.Bytes())
+		_, err = os.Stdout.Write(out)
 	} else {
-		err = ioutil.WriteFile("builtin.go", b.Bytes(), 0666)
+		err = ioutil.WriteFile("builtin.go", out, 0666)
 	}
 	if err != nil {
 		log.Fatal(err)
 	}
 }
 
-// Compile .go file, import data from .o file, and write Go string version.
 func mkbuiltin(w io.Writer, name string) {
-	args := []string{"tool", "compile", "-A"}
-	if name == "runtime" {
-		args = append(args, "-u")
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, filepath.Join("builtin", name+".go"), nil, 0)
+	if err != nil {
+		log.Fatal(err)
 	}
-	args = append(args, "builtin/"+name+".go")
 
-	if err := exec.Command("go", args...).Run(); err != nil {
-		log.Fatal(err)
+	var interner typeInterner
+
+	fmt.Fprintf(w, "var %sDecls = [...]struct { name string; tag int; typ int }{\n", name)
+	for _, decl := range f.Decls {
+		switch decl := decl.(type) {
+		case *ast.FuncDecl:
+			if decl.Recv != nil {
+				log.Fatal("methods unsupported")
+			}
+			if decl.Body != nil {
+				log.Fatal("unexpected function body")
+			}
+			fmt.Fprintf(w, "{%q, funcTag, %d},\n", decl.Name.Name, interner.intern(decl.Type))
+		case *ast.GenDecl:
+			if decl.Tok != token.VAR {
+				log.Fatal("unhandled declaration kind", decl.Tok)
+			}
+			for _, spec := range decl.Specs {
+				spec := spec.(*ast.ValueSpec)
+				if len(spec.Values) != 0 {
+					log.Fatal("unexpected values")
+				}
+				typ := interner.intern(spec.Type)
+				for _, name := range spec.Names {
+					fmt.Fprintf(w, "{%q, varTag, %d},\n", name.Name, typ)
+				}
+			}
+		default:
+			log.Fatal("unhandled decl type", decl)
+		}
 	}
-	obj := name + ".o"
-	defer os.Remove(obj)
+	fmt.Fprintln(w, "}")
 
-	b, err := ioutil.ReadFile(obj)
-	if err != nil {
-		log.Fatal(err)
+	fmt.Fprintln(w)
+	fmt.Fprintf(w, "func %sTypes() []*Type {\n", name)
+	fmt.Fprintf(w, "var typs [%d]*Type\n", len(interner.typs))
+	for i, typ := range interner.typs {
+		fmt.Fprintf(w, "typs[%d] = %s\n", i, typ)
+	}
+	fmt.Fprintln(w, "return typs[:]")
+	fmt.Fprintln(w, "}")
+}
+
+// typeInterner maps Go type expressions to compiler code that
+// constructs the denoted type. It recognizes and reuses common
+// subtype expressions.
+type typeInterner struct {
+	typs []string
+	hash map[string]int
+}
+
+func (i *typeInterner) intern(t ast.Expr) int {
+	x := i.mktype(t)
+	v, ok := i.hash[x]
+	if !ok {
+		v = len(i.typs)
+		if i.hash == nil {
+			i.hash = make(map[string]int)
+		}
+		i.hash[x] = v
+		i.typs = append(i.typs, x)
 	}
+	return v
+}
+
+func (i *typeInterner) subtype(t ast.Expr) string {
+	return fmt.Sprintf("typs[%d]", i.intern(t))
+}
+
+func (i *typeInterner) mktype(t ast.Expr) string {
+	switch t := t.(type) {
+	case *ast.Ident:
+		switch t.Name {
+		case "byte":
+			return "bytetype"
+		case "rune":
+			return "runetype"
+		}
+		return fmt.Sprintf("Types[T%s]", strings.ToUpper(t.Name))
+
+	case *ast.ArrayType:
+		if t.Len == nil {
+			return fmt.Sprintf("typSlice(%s)", i.subtype(t.Elt))
+		}
+		return fmt.Sprintf("typArray(%s, %d)", i.subtype(t.Elt), intconst(t.Len))
+	case *ast.ChanType:
+		dir := "Cboth"
+		switch t.Dir {
+		case ast.SEND:
+			dir = "Csend"
+		case ast.RECV:
+			dir = "Crecv"
+		}
+		return fmt.Sprintf("typChan(%s, %s)", i.subtype(t.Value), dir)
+	case *ast.FuncType:
+		return fmt.Sprintf("functype(nil, %s, %s)", i.fields(t.Params, false), i.fields(t.Results, false))
+	case *ast.InterfaceType:
+		if len(t.Methods.List) != 0 {
+			log.Fatal("non-empty interfaces unsupported")
+		}
+		return "Types[TINTER]"
+	case *ast.MapType:
+		return fmt.Sprintf("typMap(%s, %s)", i.subtype(t.Key), i.subtype(t.Value))
+	case *ast.StarExpr:
+		return fmt.Sprintf("typPtr(%s)", i.subtype(t.X))
+	case *ast.StructType:
+		return fmt.Sprintf("tostruct(%s)", i.fields(t.Fields, true))
 
-	// Look for $$B that introduces binary export data.
-	i := bytes.Index(b, []byte("\n$$B\n"))
-	if i < 0 {
-		log.Fatal("did not find beginning of export data")
+	default:
+		log.Fatalf("unhandled type: %#v", t)
+		panic("unreachable")
 	}
-	b = b[i+5:]
+}
 
-	// Look for $$ that closes export data.
-	i = bytes.Index(b, []byte("\n$$\n"))
-	if i < 0 {
-		log.Fatal("did not find end of export data")
+func (i *typeInterner) fields(fl *ast.FieldList, keepNames bool) string {
+	if fl == nil || len(fl.List) == 0 {
+		return "nil"
+	}
+	var res []string
+	for _, f := range fl.List {
+		typ := i.subtype(f.Type)
+		if len(f.Names) == 0 {
+			res = append(res, fmt.Sprintf("anonfield(%s)", typ))
+		} else {
+			for _, name := range f.Names {
+				if keepNames {
+					res = append(res, fmt.Sprintf("namedfield(%q, %s)", name.Name, typ))
+				} else {
+					res = append(res, fmt.Sprintf("anonfield(%s)", typ))
+				}
+			}
+		}
 	}
-	b = b[:i+4]
-
-	// Process and reformat export data.
-	const n = 40 // number of bytes per line
-	fmt.Fprintf(w, "\nconst %simport = \"\"", name)
-	for len(b) > 0 {
-		i := len(b)
-		if i > n {
-			i = n
+	return fmt.Sprintf("[]*Node{%s}", strings.Join(res, ", "))
+}
+
+func intconst(e ast.Expr) int64 {
+	switch e := e.(type) {
+	case *ast.BasicLit:
+		if e.Kind != token.INT {
+			log.Fatalf("expected INT, got %v", e.Kind)
+		}
+		x, err := strconv.ParseInt(e.Value, 0, 64)
+		if err != nil {
+			log.Fatal(err)
 		}
-		fmt.Fprintf(w, " +\n\t%q", b[:i])
-		b = b[i:]
+		return x
+	default:
+		log.Fatalf("unhandled expr: %#v", e)
+		panic("unreachable")
 	}
-	fmt.Fprintf(w, "\n")
 }