cmd/compile: enforce 32-bit restrictions on ops

Most 64-bit x86 ops can only take a signed 32-bit constant.
Clean up our rewrite rules to enforce this restriction.

Modify the assembler to fail if the offset does not fit
in the instruction.

That last check triggers a few times on weird testing code.
Suppress those errors if the compiler itself generated errors.

Fixes #14862

Change-Id: I76559af035b38483b1e59621a8029fc66b3a5d1e
Reviewed-on: https://go-review.googlesource.com/20815Reviewed-by: Josh Bleecher Snyder <josharian@gmail.com>

src/cmd/compile/internal/gc/dcl.go

@@ -1309,6 +1309,10 @@ func funccompile(n *Node) {
 	breakpc = nil
 	Funcdepth = 0
 	dclcontext = PEXTERN
+	if nerrors != 0 {
+		// If we have compile errors, ignore any assembler/linker errors.
+		Ctxt.DiagFunc = func(string, ...interface{}) {}
+	}
 	flushdata()
 	obj.Flushplist(Ctxt) // convert from Prog list to machine code
 }

src/cmd/compile/internal/ssa/rewrite.go

@@ -95,16 +95,6 @@ func typeSize(t Type) int64 {
 	return t.Size()
 }
 
-// addOff adds two int64 offsets. Fails if wraparound happens.
-func addOff(x, y int64) int64 {
-	z := x + y
-	// x and y have same sign and z has a different sign => overflow
-	if x^y >= 0 && x^z < 0 {
-		panic(fmt.Sprintf("offset overflow %d %d", x, y))
-	}
-	return z
-}
-
 // mergeSym merges two symbolic offsets. There is no real merging of
 // offsets, we just pick the non-nil one.
 func mergeSym(x, y interface{}) interface{} {

src/cmd/internal/obj/x86/asm6.go

@@ -2750,6 +2750,16 @@ func asmandsz(ctxt *obj.Link, p *obj.Prog, a *obj.Addr, r int, rex int, m64 int)
 	var rel obj.Reloc
 
 	rex &= 0x40 | Rxr
+	switch {
+	case int64(int32(a.Offset)) == a.Offset:
+		// Offset fits in sign-extended 32 bits.
+	case int64(uint32(a.Offset)) == a.Offset && ctxt.Rexflag&Rxw == 0:
+		// Offset fits in zero-extended 32 bits in a 32-bit instruction.
+		// This is allowed for assembly that wants to use 32-bit hex
+		// constants, e.g. LEAL 0x99999999(AX), AX.
+	default:
+		ctxt.Diag("offset too large in %s", p)
+	}
 	v := int32(a.Offset)
 	rel.Siz = 0