Commit fefae6ee authored by Russ Cox's avatar Russ Cox

cmd/6g, cmd/8g: move panicindex calls out of line

The old code generated for a bounds check was
                CMP
                JLT ok
                CALL panicindex
        ok:
                ...

The new code is (once the linker finishes with it):
                CMP
                JGE panic
                ...
        panic:
                CALL panicindex

which moves the calls out of line, putting more useful
code in each cache line.  This matters especially in tight
loops, such as in Fannkuch.  The benefit is more modest
elsewhere, but real.

From test/bench/go1, amd64:

benchmark                old ns/op    new ns/op    delta
BenchmarkBinaryTree17   6096092000   6088808000   -0.12%
BenchmarkFannkuch11     6151404000   4020463000  -34.64%
BenchmarkGobDecode        28990050     28894630   -0.33%
BenchmarkGobEncode        12406310     12136730   -2.17%
BenchmarkGzip               179923       179903   -0.01%
BenchmarkGunzip              11219        11130   -0.79%
BenchmarkJSONEncode       86429350     86515900   +0.10%
BenchmarkJSONDecode      334593800    315728400   -5.64%
BenchmarkRevcomp25M     1219763000   1180767000   -3.20%
BenchmarkTemplate        492947600    483646800   -1.89%

And 386:

benchmark                old ns/op    new ns/op    delta
BenchmarkBinaryTree17   6354902000   6243000000   -1.76%
BenchmarkFannkuch11     8043769000   7326965000   -8.91%
BenchmarkGobDecode        19010800     18941230   -0.37%
BenchmarkGobEncode        14077500     13792460   -2.02%
BenchmarkGzip               194087       193619   -0.24%
BenchmarkGunzip              12495        12457   -0.30%
BenchmarkJSONEncode      125636400    125451400   -0.15%
BenchmarkJSONDecode      696648600    685032800   -1.67%
BenchmarkRevcomp25M     2058088000   2052545000   -0.27%
BenchmarkTemplate        602140000    589876800   -2.04%

To implement this, two new instruction forms:

        JLT target      // same as always
        JLT $0, target  // branch expected not taken
        JLT $1, target  // branch expected taken

The linker could also emit the prediction prefixes, but it
does not: expected taken branches are reversed so that the
expected case is not taken (as in example above), and
the default expectaton for such a jump is not taken
already.

R=golang-dev, gri, r, dave
CC=golang-dev
https://golang.org/cl/6248049
parent b9918dbf
...@@ -176,6 +176,11 @@ nonrel: ...@@ -176,6 +176,11 @@ nonrel:
$$.from = nullgen; $$.from = nullgen;
$$.to = $1; $$.to = $1;
} }
| imm ',' rel
{
$$.from = $1;
$$.to = $3;
}
spec1: /* DATA */ spec1: /* DATA */
nam '/' con ',' imm nam '/' con ',' imm
......
This diff is collapsed.
...@@ -575,6 +575,7 @@ agen(Node *n, Node *res) ...@@ -575,6 +575,7 @@ agen(Node *n, Node *res)
nodconst(&n2, types[TUINT32], v); nodconst(&n2, types[TUINT32], v);
gins(optoas(OCMP, types[TUINT32]), &n1, &n2); gins(optoas(OCMP, types[TUINT32]), &n1, &n2);
p1 = gbranch(optoas(OGT, types[TUINT32]), T); p1 = gbranch(optoas(OGT, types[TUINT32]), T);
expecttaken(p1, 1);
ginscall(panicindex, 0); ginscall(panicindex, 0);
patch(p1, pc); patch(p1, pc);
} }
...@@ -625,6 +626,7 @@ agen(Node *n, Node *res) ...@@ -625,6 +626,7 @@ agen(Node *n, Node *res)
} }
gins(optoas(OCMP, t), &n2, &n1); gins(optoas(OCMP, t), &n2, &n1);
p1 = gbranch(optoas(OLT, t), T); p1 = gbranch(optoas(OLT, t), T);
expecttaken(p1, 1);
if(n5.op != OXXX) if(n5.op != OXXX)
regfree(&n5); regfree(&n5);
ginscall(panicindex, 0); ginscall(panicindex, 0);
......
...@@ -104,6 +104,7 @@ int componentgen(Node*, Node*); ...@@ -104,6 +104,7 @@ int componentgen(Node*, Node*);
void clearp(Prog*); void clearp(Prog*);
void proglist(void); void proglist(void);
Prog* gbranch(int, Type*); Prog* gbranch(int, Type*);
void expecttaken(Prog*, int);
Prog* prog(int); Prog* prog(int);
void gaddoffset(Node*); void gaddoffset(Node*);
void gconv(int, int); void gconv(int, int);
......
...@@ -508,6 +508,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res) ...@@ -508,6 +508,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res)
nodconst(&n4, t, -1); nodconst(&n4, t, -1);
gins(optoas(OCMP, t), &n3, &n4); gins(optoas(OCMP, t), &n3, &n4);
p1 = gbranch(optoas(ONE, t), T); p1 = gbranch(optoas(ONE, t), T);
expecttaken(p1, 1);
nodconst(&n4, t, -1LL<<(t->width*8-1)); nodconst(&n4, t, -1LL<<(t->width*8-1));
if(t->width == 8) { if(t->width == 8) {
n5 = n4; n5 = n4;
...@@ -516,6 +517,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res) ...@@ -516,6 +517,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res)
} }
gins(optoas(OCMP, t), &ax, &n4); gins(optoas(OCMP, t), &ax, &n4);
p2 = gbranch(optoas(ONE, t), T); p2 = gbranch(optoas(ONE, t), T);
expecttaken(p2, 1);
if(op == ODIV) if(op == ODIV)
gmove(&n4, res); gmove(&n4, res);
if(t->width == 8) if(t->width == 8)
...@@ -943,6 +945,7 @@ cgen_shift(int op, int bounded, Node *nl, Node *nr, Node *res) ...@@ -943,6 +945,7 @@ cgen_shift(int op, int bounded, Node *nl, Node *nr, Node *res)
nodconst(&n3, tcount, nl->type->width*8); nodconst(&n3, tcount, nl->type->width*8);
gins(optoas(OCMP, tcount), &n1, &n3); gins(optoas(OCMP, tcount), &n1, &n3);
p1 = gbranch(optoas(OLT, tcount), T); p1 = gbranch(optoas(OLT, tcount), T);
expecttaken(p1, 1);
if(op == ORSH && issigned[nl->type->etype]) { if(op == ORSH && issigned[nl->type->etype]) {
nodconst(&n3, types[TUINT32], nl->type->width*8-1); nodconst(&n3, types[TUINT32], nl->type->width*8-1);
gins(a, &n3, &n2); gins(a, &n3, &n2);
...@@ -1158,12 +1161,14 @@ cmpandthrow(Node *nl, Node *nr) ...@@ -1158,12 +1161,14 @@ cmpandthrow(Node *nl, Node *nr)
regfree(&n1); regfree(&n1);
if(throwpc == nil) { if(throwpc == nil) {
p1 = gbranch(optoas(op, t), T); p1 = gbranch(optoas(op, t), T);
expecttaken(p1, 1);
throwpc = pc; throwpc = pc;
ginscall(panicslice, 0); ginscall(panicslice, 0);
patch(p1, pc); patch(p1, pc);
} else { } else {
op = brcom(op); op = brcom(op);
p1 = gbranch(optoas(op, t), T); p1 = gbranch(optoas(op, t), T);
expecttaken(p1, 0);
patch(p1, throwpc); patch(p1, throwpc);
} }
} }
......
...@@ -117,6 +117,16 @@ gbranch(int as, Type *t) ...@@ -117,6 +117,16 @@ gbranch(int as, Type *t)
return p; return p;
} }
/*
* mark branch as expected taken or not.
*/
void
expecttaken(Prog *p, int taken)
{
p->from.type = D_CONST;
p->from.offset = taken;
}
/* /*
* patch previous branch to jump to to. * patch previous branch to jump to to.
*/ */
...@@ -2120,6 +2130,7 @@ oindex: ...@@ -2120,6 +2130,7 @@ oindex:
} }
gins(optoas(OCMP, t), reg1, &n2); gins(optoas(OCMP, t), reg1, &n2);
p1 = gbranch(optoas(OLT, t), T); p1 = gbranch(optoas(OLT, t), T);
expecttaken(p1, 1);
if(n4.op != OXXX) if(n4.op != OXXX)
regfree(&n4); regfree(&n4);
ginscall(panicindex, 0); ginscall(panicindex, 0);
...@@ -2229,6 +2240,7 @@ oindex_const_sudo: ...@@ -2229,6 +2240,7 @@ oindex_const_sudo:
p1 = gins(optoas(OCMP, types[TUINT32]), N, &n2); p1 = gins(optoas(OCMP, types[TUINT32]), N, &n2);
p1->from = *a; p1->from = *a;
p1 = gbranch(optoas(OGT, types[TUINT32]), T); p1 = gbranch(optoas(OGT, types[TUINT32]), T);
expecttaken(p1, 1);
ginscall(panicindex, 0); ginscall(panicindex, 0);
patch(p1, pc); patch(p1, pc);
a->offset -= Array_nel; a->offset -= Array_nel;
......
...@@ -314,7 +314,9 @@ uchar yscond[] = ...@@ -314,7 +314,9 @@ uchar yscond[] =
}; };
uchar yjcond[] = uchar yjcond[] =
{ {
Ynone, Ybr, Zbr, 1, Ynone, Ybr, Zbr, 0,
Yi0, Ybr, Zbr, 0,
Yi1, Ybr, Zbr, 1,
0 0
}; };
uchar yloop[] = uchar yloop[] =
......
...@@ -192,6 +192,18 @@ loop: ...@@ -192,6 +192,18 @@ loop:
* recurse to follow one path. * recurse to follow one path.
* continue loop on the other. * continue loop on the other.
*/ */
if(p->from.type == D_CONST) {
if(p->from.offset == 1) {
/*
* expect conditional jump to be taken.
* rewrite so that's the fall-through case.
*/
p->as = relinv(a);
q = p->link;
p->link = p->pcond;
p->pcond = q;
}
} else {
q = brchain(p->link); q = brchain(p->link);
if(q != P && q->mark) if(q != P && q->mark)
if(a != ALOOP) { if(a != ALOOP) {
...@@ -199,6 +211,7 @@ loop: ...@@ -199,6 +211,7 @@ loop:
p->link = p->pcond; p->link = p->pcond;
p->pcond = q; p->pcond = q;
} }
}
xfol(p->link, last); xfol(p->link, last);
q = brchain(p->pcond); q = brchain(p->pcond);
if(q->mark) { if(q->mark) {
...@@ -413,6 +426,14 @@ dostkoff(void) ...@@ -413,6 +426,14 @@ dostkoff(void)
if(autoffset < 0) if(autoffset < 0)
autoffset = 0; autoffset = 0;
if(autoffset < StackSmall && !(p->from.scale & NOSPLIT)) {
for(q = p; q != P; q = q->link)
if(q->as == ACALL)
goto noleaf;
p->from.scale |= NOSPLIT;
noleaf:;
}
q = P; q = P;
if((p->from.scale & NOSPLIT) && autoffset >= StackSmall) if((p->from.scale & NOSPLIT) && autoffset >= StackSmall)
diag("nosplit func likely to overflow stack"); diag("nosplit func likely to overflow stack");
......
...@@ -177,6 +177,11 @@ nonrel: ...@@ -177,6 +177,11 @@ nonrel:
$$.from = nullgen; $$.from = nullgen;
$$.to = $1; $$.to = $1;
} }
| imm ',' rel
{
$$.from = $1;
$$.to = $3;
}
spec1: /* DATA */ spec1: /* DATA */
nam '/' con ',' imm nam '/' con ',' imm
......
This diff is collapsed.
/* A Bison parser, made by GNU Bison 2.3. */
/* A Bison parser, made by GNU Bison 2.4.1. */
/* Skeleton interface for Bison's Yacc-like parsers in C /* Skeleton interface for Bison's Yacc-like parsers in C
Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006 Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
Free Software Foundation, Inc. Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation; either version 2, or (at your option)
(at your option) any later version. any later version.
This program is distributed in the hope that it will be useful, This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
...@@ -17,7 +16,9 @@ ...@@ -17,7 +16,9 @@
GNU General Public License for more details. GNU General Public License for more details.
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. */ along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor,
Boston, MA 02110-1301, USA. */
/* As a special exception, you may create a larger work that contains /* As a special exception, you may create a larger work that contains
part or all of the Bison parser skeleton and distribute that work part or all of the Bison parser skeleton and distribute that work
...@@ -32,7 +33,6 @@ ...@@ -32,7 +33,6 @@
This special exception was added by the Free Software Foundation in This special exception was added by the Free Software Foundation in
version 2.2 of Bison. */ version 2.2 of Bison. */
/* Tokens. */ /* Tokens. */
#ifndef YYTOKENTYPE #ifndef YYTOKENTYPE
# define YYTOKENTYPE # define YYTOKENTYPE
...@@ -104,11 +104,8 @@ ...@@ -104,11 +104,8 @@
#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
typedef union YYSTYPE typedef union YYSTYPE
{
/* Line 1676 of yacc.c */
#line 37 "a.y" #line 37 "a.y"
{
Sym *sym; Sym *sym;
int32 lval; int32 lval;
struct { struct {
...@@ -119,17 +116,14 @@ typedef union YYSTYPE ...@@ -119,17 +116,14 @@ typedef union YYSTYPE
char sval[8]; char sval[8];
Gen gen; Gen gen;
Gen2 gen2; Gen2 gen2;
}
/* Line 1529 of yacc.c. */
#line 122 "y.tab.h"
/* Line 1676 of yacc.c */ YYSTYPE;
#line 127 "y.tab.h"
} YYSTYPE;
# define YYSTYPE_IS_TRIVIAL 1
# define yystype YYSTYPE /* obsolescent; will be withdrawn */ # define yystype YYSTYPE /* obsolescent; will be withdrawn */
# define YYSTYPE_IS_DECLARED 1 # define YYSTYPE_IS_DECLARED 1
# define YYSTYPE_IS_TRIVIAL 1
#endif #endif
extern YYSTYPE yylval; extern YYSTYPE yylval;
...@@ -596,6 +596,7 @@ agen(Node *n, Node *res) ...@@ -596,6 +596,7 @@ agen(Node *n, Node *res)
nodconst(&n2, types[TUINT32], v); nodconst(&n2, types[TUINT32], v);
gins(optoas(OCMP, types[TUINT32]), &n1, &n2); gins(optoas(OCMP, types[TUINT32]), &n1, &n2);
p1 = gbranch(optoas(OGT, types[TUINT32]), T); p1 = gbranch(optoas(OGT, types[TUINT32]), T);
expecttaken(p1, 1);
ginscall(panicindex, 0); ginscall(panicindex, 0);
patch(p1, pc); patch(p1, pc);
} }
...@@ -633,6 +634,7 @@ agen(Node *n, Node *res) ...@@ -633,6 +634,7 @@ agen(Node *n, Node *res)
nodconst(&n1, types[TUINT32], nl->type->bound); nodconst(&n1, types[TUINT32], nl->type->bound);
gins(optoas(OCMP, types[TUINT32]), &n2, &n1); gins(optoas(OCMP, types[TUINT32]), &n2, &n1);
p1 = gbranch(optoas(OLT, types[TUINT32]), T); p1 = gbranch(optoas(OLT, types[TUINT32]), T);
expecttaken(p1, 1);
if(p2) if(p2)
patch(p2, pc); patch(p2, pc);
ginscall(panicindex, 0); ginscall(panicindex, 0);
......
...@@ -122,6 +122,7 @@ void cgen64(Node*, Node*); ...@@ -122,6 +122,7 @@ void cgen64(Node*, Node*);
void clearp(Prog*); void clearp(Prog*);
void proglist(void); void proglist(void);
Prog* gbranch(int, Type*); Prog* gbranch(int, Type*);
void expecttaken(Prog*, int);
Prog* prog(int); Prog* prog(int);
void gaddoffset(Node*); void gaddoffset(Node*);
void gconv(int, int); void gconv(int, int);
......
...@@ -896,12 +896,14 @@ cmpandthrow(Node *nl, Node *nr) ...@@ -896,12 +896,14 @@ cmpandthrow(Node *nl, Node *nr)
regfree(&n1); regfree(&n1);
if(throwpc == nil) { if(throwpc == nil) {
p1 = gbranch(optoas(op, t), T); p1 = gbranch(optoas(op, t), T);
expecttaken(p1, 1);
throwpc = pc; throwpc = pc;
ginscall(panicslice, 0); ginscall(panicslice, 0);
patch(p1, pc); patch(p1, pc);
} else { } else {
op = brcom(op); op = brcom(op);
p1 = gbranch(optoas(op, t), T); p1 = gbranch(optoas(op, t), T);
expecttaken(p1, 0);
patch(p1, throwpc); patch(p1, throwpc);
} }
} }
......
...@@ -118,6 +118,13 @@ gbranch(int as, Type *t) ...@@ -118,6 +118,13 @@ gbranch(int as, Type *t)
return p; return p;
} }
void
expecttaken(Prog *p, int taken)
{
p->from.type = D_CONST;
p->from.offset = taken;
}
/* /*
* patch previous branch to jump to to. * patch previous branch to jump to to.
*/ */
......
...@@ -254,7 +254,9 @@ uchar yscond[] = ...@@ -254,7 +254,9 @@ uchar yscond[] =
}; };
uchar yjcond[] = uchar yjcond[] =
{ {
Ynone, Ybr, Zbr, 1, Ynone, Ybr, Zbr, 0,
Yi0, Ybr, Zbr, 0,
Yi1, Ybr, Zbr, 1,
0 0
}; };
uchar yloop[] = uchar yloop[] =
......
...@@ -184,6 +184,18 @@ loop: ...@@ -184,6 +184,18 @@ loop:
* recurse to follow one path. * recurse to follow one path.
* continue loop on the other. * continue loop on the other.
*/ */
if(p->from.type == D_CONST) {
if(p->from.offset == 1) {
/*
* expect conditional jump to be taken.
* rewrite so that's the fall-through case.
*/
p->as = relinv(a);
q = p->link;
p->link = p->pcond;
p->pcond = q;
}
} else {
q = brchain(p->link); q = brchain(p->link);
if(q != P && q->mark) if(q != P && q->mark)
if(a != ALOOP) { if(a != ALOOP) {
...@@ -191,6 +203,7 @@ loop: ...@@ -191,6 +203,7 @@ loop:
p->link = p->pcond; p->link = p->pcond;
p->pcond = q; p->pcond = q;
} }
}
xfol(p->link, last); xfol(p->link, last);
q = brchain(p->pcond); q = brchain(p->pcond);
if(q->mark) { if(q->mark) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment