cmd/compile: switch to compact export format by default

builtin.go was auto-generated via go generate; all other
changes were manual.

The new format reduces the export data size by ~65% on average
for the std library packages (and there is still quite a bit of
room for improvement).

The average time to write export data is reduced by (at least)
62% as measured in one run over the std lib, it is likely more.

The average time to read import data is reduced by (at least)
37% as measured in one run over the std lib, it is likely more.
There is also room to improve this time.

The compiler transparently handles both packages using the old
and the new format.

Comparing the -S output of the go build for each package via
the cmp.bash script (added) shows identical assembly code for
all packages, but 6 files show file:line differences:

The following files have differences because they use cgo
and cgo uses different temp. directories for different builds.
Harmless.

	src/crypto/x509
	src/net
	src/os/user
	src/runtime/cgo

The following files have file:line differences that are not yet
fully explained; however the differences exist w/ and w/o new export
format (pre-existing condition). See issue #15453.

	src/go/internal/gccgoimporter
	src/go/internal/gcimporter

In summary, switching to the new export format produces the same
package files as before for all practical purposes.

How can you tell which one you have (if you care): Open a package
(.a) file in an editor. Textual export data starts with a $$ after
the header and is more or less legible; binary export data starts
with a $$B after the header and is mostly unreadable. A stand-alone
decoder (for debugging) is in the works.

In case of a problem, please first try reverting back to the old
textual format to determine if the cause is the new export format:

For a stand-alone compiler invocation:
- go tool compile -newexport=0 <files>

For a single package:
- go build -gcflags="-newexport=0" <pkg>

For make/all.bash:
- (export GO_GCFLAGS="-newexport=0"; sh make.bash)

Fixes #13241.

Change-Id: I2588cb463be80af22446bf80c225e92ab79878b8
Reviewed-on: https://go-review.googlesource.com/22123Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Run-TryBot: Robert Griesemer <gri@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>

src/cmd/compile/internal/gc/bexport.go

@@ -112,9 +112,6 @@ import (
 // (suspected) format errors, and whenever a change is made to the format.
 const debugFormat = false // default: false
 
-// TODO(gri) remove eventually
-const forceNewExport = false // force new export format - do NOT submit with this flag set
-
 // forceObjFileStability enforces additional constraints in export data
 // and other parts of the compiler to eliminate object file differences
 // only due to the choice of export format.

src/cmd/compile/internal/gc/export.go

@@ -377,7 +377,7 @@ func dumpexport() {
 	}
 
 	size := 0 // size of export section without enclosing markers
-	if forceNewExport || newexport {
+	if newexport {
 		// binary export
 		// The linker also looks for the $$ marker - use char after $$ to distinguish format.
 		exportf("\n$$B\n") // indicate binary format

src/cmd/compile/internal/gc/main.go

@@ -181,7 +181,7 @@ func Main() {
 	obj.Flagcount("live", "debug liveness analysis", &debuglive)
 	obj.Flagcount("m", "print optimization decisions", &Debug['m'])
 	flag.BoolVar(&flag_msan, "msan", false, "build code compatible with C/C++ memory sanitizer")
-	flag.BoolVar(&newexport, "newexport", false, "use new export format") // TODO(gri) remove eventually (issue 13241)
+	flag.BoolVar(&newexport, "newexport", true, "use new export format") // TODO(gri) remove eventually (issue 15323)
 	flag.BoolVar(&nolocalimports, "nolocalimports", false, "reject local (relative) imports")
 	flag.StringVar(&outfile, "o", "", "write output to `file`")
 	flag.StringVar(&myimportpath, "p", "", "set expected package import `path`")

src/cmp.bash

+#!/usr/bin/env bash
+
+# Copyright 2016 The Go Authors.  All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# A simple script to compare differences between
+# assembly listings for packages built with different
+# compiler flags. It is useful to inspect the impact
+# of a compiler change across all std lib packages.
+#
+# The script builds the std library (make.bash) once
+# with FLAGS1 and once with FLAGS2 and compares the
+# "go build <pkg>" assembly output for each package
+# and lists the packages with differences.
+#
+# It leaves and old.txt and new.txt file in the package
+# directories for the packages with differences.
+
+FLAGS1="-newexport=0"
+FLAGS2="-newexport=1"
+
+echo
+echo
+echo "1a) clean build using $FLAGS1"
+(export GO_GCFLAGS="$FLAGS1"; sh make.bash)
+
+echo
+echo
+echo "1b) save go build output for all packages"
+for pkg in `go list std`; do
+	echo $pkg
+	DIR=$GOROOT/src/$pkg
+	go build -gcflags "$FLAGS1 -S" -o /dev/null $pkg &> $DIR/old.txt
+done
+
+echo
+echo
+echo "2a) clean build using $FLAGS2"
+(export GO_GCFLAGS="$FLAGS2"; sh make.bash)
+
+echo
+echo
+echo "2b) save go build output for all packages"
+for pkg in `go list std`; do
+	echo $pkg
+	DIR=$GOROOT/src/$pkg
+	go build -gcflags "$FLAGS2 -S" -o /dev/null $pkg &> $DIR/new.txt
+done
+
+echo
+echo
+echo "3) compare assembly files"
+for pkg in `go list std`; do
+	DIR=$GOROOT/src/$pkg
+
+	if cmp $DIR/old.txt $DIR/new.txt &> /dev/null
+	then rm $DIR/old.txt $DIR/new.txt
+	else echo "==> $DIR"
+	fi
+done