cmd/ld: use native-endian symbol values in symbol table

The Plan 9 symbol table format defines big-endian symbol values
for portability, but we want to be able to generate an ELF object file
and let the host linker link it, as part of the solution to issue 4069.
The symbol table itself, since it is loaded into memory at run time,
must be filled in by the final host linker, using relocation directives
to set the symbol values. On a little-endian machine, the linker will
only fill in little-endian values during relocation, so we are forced
to use little-endian symbol values.

To preserve most of the original portability of the symbol table
format, we make the table itself say whether it uses big- or
little-endian values. If the table begins with the magic sequence
        fe ff ff ff 00 00
then the actual table begins after those six bytes and contains
little-endian symbol values. Otherwise, the table is in the original
format and contains big-endian symbol values. The magic sequence
looks like an "end of table" entry (the fifth byte is zero), so legacy
readers will see a little-endian table as an empty table.

All the gc architectures are little-endian today, so the practical
effect of this CL is to make all the generated tables little-endian,
but if a big-endian system comes along, ld will not generate
the magic sequence, and the various readers will fall back to the
original big-endian interpretation.

R=ken2
CC=golang-dev
https://golang.org/cl/7066043

doc/go1.1.html

@@ -66,6 +66,15 @@ to adjust frame pointer offsets.
 The implementation now includes a built-in <a href="/doc/articles/race_detector.html">data race detector</a>.
 </p>
 
+<h3 id="symtab">Symbol table</h3>
+
+<p>
+In the gc toolchain, the symbol table format has been extended to allow
+little-endian encoding of symbol values, and the extension is used in
+binaries generated by the Go 1.1 version of the gc linker.
+To the Go 1.0 toolchain and libraries, these new symbol tables appear empty.
+</p>
+
 <h2 id="library">Changes to the standard library</h2>
 
 <h3 id="debug/elf">debug/elf</h3>

src/cmd/ld/data.c

@@ -150,14 +150,14 @@ relocsym(Sym *s)
 	int32 i, off, siz, fl;
 	vlong o;
 	uchar *cast;
-	
+
 	cursym = s;
 	memset(&p, 0, sizeof p);
 	for(r=s->r; r<s->r+s->nr; r++) {
 		off = r->off;
 		siz = r->siz;
-		if(off < 0 || off+(siz&~Rbig) > s->np) {
-			diag("%s: invalid relocation %d+%d not in [%d,%d)", s->name, off, siz&~Rbig, 0, s->np);
+		if(off < 0 || off+siz > s->np) {
+			diag("%s: invalid relocation %d+%d not in [%d,%d)", s->name, off, siz, 0, s->np);
 			continue;
 		}
 		if(r->sym != S && (r->sym->type & SMASK == 0 || r->sym->type & SMASK == SXREF)) {
@@ -198,20 +198,6 @@ relocsym(Sym *s)
 		default:
 			cursym = s;
 			diag("bad reloc size %#ux for %s", siz, r->sym->name);
-		case 4 + Rbig:
-			fl = o;
-			s->p[off] = fl>>24;
-			s->p[off+1] = fl>>16;
-			s->p[off+2] = fl>>8;
-			s->p[off+3] = fl;
-			break;
-		case 4 + Rlittle:
-			fl = o;
-			s->p[off] = fl;
-			s->p[off+1] = fl>>8;
-			s->p[off+2] = fl>>16;
-			s->p[off+3] = fl>>24;
-			break;
 		case 4:
 			fl = o;
 			cast = (uchar*)&fl;
@@ -223,7 +209,7 @@ relocsym(Sym *s)
 			for(i=0; i<8; i++)
 				s->p[off+i] = cast[inuxi8[i]];
 			break;
-		}		
+		}
 	}
 }
 
@@ -231,7 +217,7 @@ void
 reloc(void)
 {
 	Sym *s;
-	
+
 	if(debug['v'])
 		Bprint(&bso, "%5.2f reloc\n", cputime());
 	Bflush(&bso);
@@ -246,10 +232,10 @@ void
 dynrelocsym(Sym *s)
 {
 	Reloc *r;
-	
+
 	if(HEADTYPE == Hwindows) {
 		Sym *rel, *targ;
-		
+
 		rel = lookup(".rel", 0);
 		if(s == rel)
 			return;
@@ -259,7 +245,7 @@ dynrelocsym(Sym *s)
 				targ->plt = rel->size;
 				r->sym = rel;
 				r->add = targ->plt;
-				
+
 				// jmp *addr
 				if(thechar == '8') {
 					adduint8(rel, 0xff);
@@ -291,7 +277,7 @@ void
 dynreloc(void)
 {
 	Sym *s;
-	
+
 	// -d supresses dynamic loader format, so we may as well not
 	// compute these sections or mark their symbols as reachable.
 	if(debug['d'] && HEADTYPE != Hwindows)
@@ -364,12 +350,12 @@ savedata(Sym *s, Prog *p, char *pn)
 			break;
 		}
 		break;
-	
+
 	case D_SCONST:
 		for(i=0; i<siz; i++)
 			s->p[off+i] = p->to.scon[i];
 		break;
-	
+
 	case D_CONST:
 		if(p->to.sym)
 			goto Addr;
@@ -450,12 +436,12 @@ blk(Sym *start, int32 addr, int32 size)
 			errorexit();
 		}
 	}
-	
+
 	for(; addr < eaddr; addr++)
 		cput(0);
 	cflush();
 }
-			
+
 void
 codeblk(int32 addr, int32 size)
 {
@@ -498,7 +484,7 @@ codeblk(int32 addr, int32 size)
 			Bprint(&bso, "%.6llux\t%-20s | foreign text\n", (vlong)addr, sym->name);
 			n = sym->size;
 			q = sym->p;
-			
+
 			while(n >= 16) {
 				Bprint(&bso, "%.6ux\t%-20.16I\n", addr, q);
 				addr += 16;
@@ -510,7 +496,7 @@ codeblk(int32 addr, int32 size)
 			addr += n;
 			continue;
 		}
-			
+
 		Bprint(&bso, "%.6llux\t%-20s | %P\n", (vlong)sym->value, sym->name, p);
 		for(p = p->link; p != P; p = p->link) {
 			if(p->link != P)
@@ -532,7 +518,7 @@ codeblk(int32 addr, int32 size)
 	}
 	Bflush(&bso);
 }
-			
+
 void
 datblk(int32 addr, int32 size)
 {
@@ -595,7 +581,7 @@ addstrdata(char *name, char *value)
 {
 	Sym *s, *sp;
 	char *p;
-	
+
 	p = smprint("%s.str", name);
 	sp = lookup(p, 0);
 	free(p);
@@ -775,7 +761,7 @@ addpcrelplus(Sym *s, Sym *t, int32 add)
 {
 	vlong i;
 	Reloc *r;
-	
+
 	if(s->type == 0)
 		s->type = SDATA;
 	s->reachable = 1;
@@ -972,7 +958,7 @@ dodata(void)
 	 * symbol, which is itself data.
 	 */
 	dynreloc();
-	
+
 	/* some symbols may no longer belong in datap (Mach-O) */
 	for(l=&datap; (s=*l) != nil; ) {
 		if(s->type <= STEXT || SXREF <= s->type)
@@ -1099,7 +1085,7 @@ dodata(void)
 	}
 	sect->len = datsize - sect->vaddr;
 	datsize = rnd(datsize, PtrSize);
-	
+
 	/* gcdata */
 	sect = addsection(&segtext, ".gcdata", 04);
 	sect->vaddr = datsize;
@@ -1169,7 +1155,7 @@ textaddress(void)
 	addsection(&segtext, ".text", 05);
 
 	// Assign PCs in text segment.
-	// Could parallelize, by assigning to text 
+	// Could parallelize, by assigning to text
 	// and then letting threads copy down, but probably not worth it.
 	sect = segtext.sect;
 	va = INITTEXT;
@@ -1192,7 +1178,7 @@ textaddress(void)
 		}
 		va += sym->size;
 	}
-	
+
 	// Align end of code so that rodata starts aligned.
 	// 128 bytes is likely overkill but definitely cheap.
 	va = rnd(va, 128);
@@ -1267,7 +1253,7 @@ address(void)
 		for(sub = sym->sub; sub != nil; sub = sub->sub)
 			sub->value += sym->value;
 	}
-	
+
 	xdefine("text", STEXT, text->vaddr);
 	xdefine("etext", STEXT, text->vaddr + text->len);
 	xdefine("rodata", SRODATA, rodata->vaddr);

src/cmd/ld/lib.h

@@ -31,7 +31,7 @@
 enum
 {
 	Sxxx,
-	
+
 	/* order here is order in output file */
 	STEXT,
 	SMACHOPLT,
@@ -246,12 +246,6 @@ struct Endian
 
 extern Endian be, le;
 
-// relocation size bits
-enum {
-	Rbig = 128,
-	Rlittle = 64,
-};
-
 /* set by call to mywhatsys() */
 extern	char*	goroot;
 extern	char*	goarch;

src/cmd/ld/symtab.c

@@ -199,7 +199,7 @@ static void
 slputb(int32 v)
 {
 	uchar *p;
-	
+
 	symgrow(symt, symt->size+4);
 	p = symt->p + symt->size;
 	*p++ = v>>24;
@@ -209,6 +209,22 @@ slputb(int32 v)
 	symt->size += 4;
 }
 
+static void
+slputl(int32 v)
+{
+	uchar *p;
+
+	symgrow(symt, symt->size+4);
+	p = symt->p + symt->size;
+	*p++ = v;
+	*p++ = v>>8;
+	*p++ = v>>16;
+	*p = v>>24;
+	symt->size += 4;
+}
+
+static void (*slput)(int32);
+
 void
 wputl(ushort w)
 {
@@ -269,15 +285,24 @@ putsymb(Sym *s, char *name, int t, vlong v, vlong size, int ver, Sym *typ)
 //		l = 8;
 	if(s != nil) {
 		rel = addrel(symt);
-		rel->siz = l + Rbig;
+		rel->siz = l;
 		rel->sym = s;
 		rel->type = D_ADDR;
 		rel->off = symt->size;
 		v = 0;
-	}	
-	if(l == 8)
-		slputb(v>>32);
-	slputb(v);
+	}
+
+	if(l == 8) {
+		if(slput == slputl) {
+			slputl(v);
+			slputl(v>>32);
+		} else {
+			slputb(v>>32);
+			slputb(v);
+		}
+	} else
+		slput(v);
+
 	if(ver)
 		t += 'a' - 'A';
 	scput(t+0x80);			/* 0x80 is variable length */
@@ -306,8 +331,8 @@ putsymb(Sym *s, char *name, int t, vlong v, vlong size, int ver, Sym *typ)
 		rel->off = symt->size;
 	}
 	if(l == 8)
-		slputb(0);
-	slputb(0);
+		slput(0);
+	slput(0);
 
 	if(debug['n']) {
 		if(t == 'z' || t == 'Z') {
@@ -356,7 +381,7 @@ symtab(void)
 	xdefine("end", SBSS, 0);
 	xdefine("epclntab", SRODATA, 0);
 	xdefine("esymtab", SRODATA, 0);
-	
+
 	// pseudo-symbols to mark locations of type, string, and go string data.
 	s = lookup("type.*", 0);
 	s->type = STYPE;
@@ -372,7 +397,7 @@ symtab(void)
 	symt->type = SSYMTAB;
 	symt->size = 0;
 	symt->reachable = 1;
-	
+
 	// assign specific types so that they sort together.
 	// within a type they sort by size, so the .* symbols
 	// just defined above will be first.
@@ -396,5 +421,25 @@ symtab(void)
 
 	if(debug['s'])
 		return;
+
+	switch(thechar) {
+	default:
+		diag("unknown architecture %c", thechar);
+		errorexit();
+	case '5':
+	case '6':
+	case '8':
+		// magic entry to denote little-endian symbol table
+		slputl(0xfffffffe);
+		scput(0);
+		scput(0);
+		slput = slputl;
+		break;
+	case 'v':
+		// big-endian (in case one comes along)
+		slput = slputb;
+		break;
+	}
+
 	genasmsym(putsymb);
 }

src/libmach/sym.c

 // Inferno libmach/sym.c
 // http://code.google.com/p/inferno-os/source/browse/utils/libmach/sym.c
 //
-// 	Copyright © 1994-1999 Lucent Technologies Inc.
-// 	Power PC support Copyright © 1995-2004 C H Forsyth (forsyth@terzarima.net).
-// 	Portions Copyright © 1997-1999 Vita Nuova Limited.
-// 	Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com).
-// 	Revisions Copyright © 2000-2004 Lucent Technologies Inc. and others.
+//	Copyright © 1994-1999 Lucent Technologies Inc.
+//	Power PC support Copyright © 1995-2004 C H Forsyth (forsyth@terzarima.net).
+//	Portions Copyright © 1997-1999 Vita Nuova Limited.
+//	Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com).
+//	Revisions Copyright © 2000-2004 Lucent Technologies Inc. and others.
 //	Portions Copyright © 2009 The Go Authors.  All rights reserved.
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -113,12 +113,18 @@ syminit(int fd, Fhdr *fp)
 	vlong vl;
 	Biobuf b;
 	int svalsz;
+	uvlong (*swav)(uvlong);
+	uint32 (*swal)(uint32);
+	uchar buf[6];
 
 	if(fp->symsz == 0)
 		return 0;
 	if(fp->type == FNONE)
 		return 0;
 
+	swav = beswav;
+	swal = beswal;
+
 	cleansyms();
 	textseg(fp->txtaddr, fp);
 		/* minimum symbol record size = 4+1+2 bytes */
@@ -129,6 +135,15 @@ syminit(int fd, Fhdr *fp)
 	}
 	Binit(&b, fd, OREAD);
 	Bseek(&b, fp->symoff, 0);
+	memset(buf, 0, sizeof buf);
+	Bread(&b, buf, sizeof buf);
+	if(memcmp(buf, "\xfe\xff\xff\xff\x00\x00", 6) == 0) {
+		swav = leswav;
+		swal = leswal;
+	} else {
+		Bseek(&b, fp->symoff, 0);
+	}
+
 	nsym = 0;
 	size = 0;
 	for(p = symbols; size < fp->symsz; p++, nsym++) {
@@ -136,13 +151,13 @@ syminit(int fd, Fhdr *fp)
 			svalsz = 8;
 			if(Bread(&b, &vl, 8) != 8)
 				return symerrmsg(8, "symbol");
-			p->value = beswav(vl);
+			p->value = swav(vl);
 		}
 		else{
 			svalsz = 4;
 			if(Bread(&b, &l, 4) != 4)
 				return symerrmsg(4, "symbol");
-			p->value = (u32int)beswal(l);
+			p->value = (u32int)swal(l);
 		}
 		if(Bread(&b, &p->type, sizeof(p->type)) != sizeof(p->type))
 			return symerrmsg(sizeof(p->value), "symbol");
@@ -155,12 +170,12 @@ syminit(int fd, Fhdr *fp)
 		if(svalsz == 8){
 			if(Bread(&b, &vl, 8) != 8)
 				return symerrmsg(8, "symbol");
-			p->gotype = beswav(vl);
+			p->gotype = swav(vl);
 		}
 		else{
 			if(Bread(&b, &l, 4) != 4)
 				return symerrmsg(4, "symbol");
-			p->gotype = (u32int)beswal(l);
+			p->gotype = (u32int)swal(l);
 		}
 		size += svalsz;
 

src/pkg/debug/gosym/pclntab_test.go

@@ -129,7 +129,7 @@ func TestLineFromAline(t *testing.T) {
 		if !ok {
 			t.Errorf("file %s starts on line %d", path, line)
 		} else if line != ll+1 {
-			t.Errorf("expected next line of file %s to be %d, got %d", path, ll+1, line)
+			t.Fatalf("expected next line of file %s to be %d, got %d", path, ll+1, line)
 		}
 		lastline[path] = line
 	}

src/pkg/debug/gosym/symtab.go

@@ -13,6 +13,7 @@ package gosym
 // and the Go format is the runtime source, specifically ../../runtime/symtab.c.
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
 	"strconv"
@@ -104,11 +105,18 @@ type sym struct {
 	name   []byte
 }
 
+var littleEndianSymtab = []byte{0xFE, 0xFF, 0xFF, 0xFF, 0x00, 0x00}
+
 func walksymtab(data []byte, fn func(sym) error) error {
+	var order binary.ByteOrder = binary.BigEndian
+	if bytes.HasPrefix(data, littleEndianSymtab) {
+		data = data[6:]
+		order = binary.LittleEndian
+	}
 	var s sym
 	p := data
 	for len(p) >= 6 {
-		s.value = binary.BigEndian.Uint32(p[0:4])
+		s.value = order.Uint32(p[0:4])
 		typ := p[4]
 		if typ&0x80 == 0 {
 			return &DecodingError{len(data) - len(p) + 4, "bad symbol type", typ}
@@ -139,7 +147,7 @@ func walksymtab(data []byte, fn func(sym) error) error {
 		}
 		s.name = p[0:i]
 		i += nnul
-		s.gotype = binary.BigEndian.Uint32(p[i : i+4])
+		s.gotype = order.Uint32(p[i : i+4])
 		p = p[i+4:]
 		fn(s)
 	}

src/pkg/runtime/symtab.c

@@ -40,13 +40,26 @@ walksymtab(void (*fn)(Sym*))
 {
 	byte *p, *ep, *q;
 	Sym s;
+	int32 bigend;
 
 	p = symtab;
 	ep = esymtab;
+
+	// Default is big-endian value encoding.
+	// If table begins fe ff ff ff 00 00, little-endian.
+	bigend = 1;
+	if(symtab[0] == 0xfe && symtab[1] == 0xff && symtab[2] == 0xff && symtab[3] == 0xff && symtab[4] == 0x00 && symtab[5] == 0x00) {
+		p += 6;
+		bigend = 0;
+	}
 	while(p < ep) {
 		if(p + 7 > ep)
 			break;
-		s.value = ((uint32)p[0]<<24) | ((uint32)p[1]<<16) | ((uint32)p[2]<<8) | ((uint32)p[3]);
+
+		if(bigend)
+			s.value = ((uint32)p[0]<<24) | ((uint32)p[1]<<16) | ((uint32)p[2]<<8) | ((uint32)p[3]);
+		else
+			s.value = ((uint32)p[3]<<24) | ((uint32)p[2]<<16) | ((uint32)p[1]<<8) | ((uint32)p[0]);
 
 		if(!(p[4]&0x80))
 			break;
@@ -304,7 +317,7 @@ splitpcln(void)
 			line += *p++;
 		else
 			line -= *p++ - 64;
-		
+
 		// pc, line now match.
 		// Because the state machine begins at pc==entry and line==0,
 		// it can happen - just at the beginning! - that the update may
@@ -326,7 +339,7 @@ splitpcln(void)
 			while(f < ef && pc >= (f+1)->entry);
 			f->pcln.array = p;
 			// pc0 and ln0 are the starting values for
-			// the loop over f->pcln, so pc must be 
+			// the loop over f->pcln, so pc must be
 			// adjusted by the same pcquant update
 			// that we're going to do as we continue our loop.
 			f->pc0 = pc + pcquant;
@@ -352,11 +365,11 @@ runtime·funcline(Func *f, uintptr targetpc)
 	uintptr pc;
 	int32 line;
 	int32 pcquant;
-	
+
 	enum {
 		debug = 0
 	};
-	
+
 	switch(thechar) {
 	case '5':
 		pcquant = 4;
@@ -383,7 +396,7 @@ runtime·funcline(Func *f, uintptr targetpc)
 
 		if(debug && !runtime·panicking)
 			runtime·printf("pc<%p targetpc=%p line=%d\n", pc, targetpc, line);
-		
+
 		// If the pc has advanced too far or we're out of data,
 		// stop and the last known line number.
 		if(pc > targetpc || p >= ep)
@@ -519,7 +532,7 @@ static bool
 hasprefix(String s, int8 *p)
 {
 	int32 i;
-	
+
 	for(i=0; i<s.len; i++) {
 		if(p[i] == 0)
 			return 1;
@@ -533,7 +546,7 @@ static bool
 contains(String s, int8 *p)
 {
 	int32 i;
-	
+
 	if(p[0] == 0)
 		return 1;
 	for(i=0; i<s.len; i++) {
@@ -549,7 +562,7 @@ bool
 runtime·showframe(Func *f)
 {
 	static int32 traceback = -1;
-	
+
 	if(traceback < 0)
 		traceback = runtime·gotraceback();
 	return traceback > 1 || contains(f->name, ".") && !hasprefix(f->name, "runtime.");