Commit c510bb9e authored by Kevin Modzelewski's avatar Kevin Modzelewski

I guess sre_parse.parse is slow

parent 412730c4
import sre_compile
import sre_constants
import sre_parse
import re
IN = sre_constants.IN
......@@ -7,26 +8,21 @@ IN = sre_constants.IN
def _identity(x):
return x
_cache = {}
DEBUG = 1
def _compile(*key):
# print "re._compile", key
# internal: compile pattern
pattern, flags = key
bypass_cache = flags & DEBUG
if not bypass_cache:
cachekey = (type(key[0]),) + key
p = _cache.get(cachekey)
if p is not None:
# print "got from cache"
return p
p = (1, 0)
_cache[cachekey] = p
return p
for i in xrange(1000000):
# sre_compile._optimize_charset([('negate', None), ('literal', 34), ('literal', 92)], _identity)
# sre_compile._compile([17, 4, 0, 3, 0, 29, 12, 0, 4294967295L, 15, 7, 26, 19, 34, 19, 92, 0, 1, 28, 0, 0, 4294967295L, 7, 6, 19, 92, 2, 18, 15, 13, 19, 34, 5, 7, 0, 19, 34, 19, 34, 1, 18, 2, 0, 29, 0, 0, 4294967295L], [(IN, [('negate', None), ('literal', 34), ('literal', 92)])], 0)
# sre_compile._compile([17, 8, 3, 1, 1, 1, 1, 97, 0], [('literal', 97)], 0)
_compile("a", 0)
pass
# This is the string tokenizer.PseudoToken:
pattern = '[ \\f\\t]*((\\\\\\r?\\n|\\Z|#[^\\r\\n]*|([uUbB]?[rR]?\'\'\'|[uUbB]?[rR]?"""))|((\\d+[jJ]|((\\d+\\.\\d*|\\.\\d+)([eE][-+]?\\d+)?|\\d+[eE][-+]?\\d+)[jJ])|((\\d+\\.\\d*|\\.\\d+)([eE][-+]?\\d+)?|\\d+[eE][-+]?\\d+)|(0[xX][\\da-fA-F]+[lL]?|0[bB][01]+[lL]?|(0[oO][0-7]+)|(0[0-7]*)[lL]?|[1-9]\\d*[lL]?))|((\\*\\*=?|>>=?|<<=?|<>|!=|//=?|[+\\-*/%&|^=<>]=?|~)|[][(){}]|(\\r?\\n|[:;.,`@]))|([uUbB]?[rR]?\'[^\\n\'\\\\]*(?:\\\\.[^\\n\'\\\\]*)*(\'|\\\\\\r?\\n)|[uUbB]?[rR]?"[^\\n"\\\\]*(?:\\\\.[^\\n"\\\\]*)*("|\\\\\\r?\\n))|[a-zA-Z_]\\w*)'
for i in xrange(100):
# re.compile checks if the pattern is in the cache, and then calls sre_compile:
sre_compile.compile(pattern, 0)
for i in xrange(600):
p = sre_parse.Pattern()
p.flags = 0
p.str = pattern
sre_parse._parse_sub(sre_parse.Tokenizer(pattern), p, 0)
......@@ -6,7 +6,7 @@ if __name__ == "__main__":
parser.add_argument("tracebacks_file", action="store", default=None)
parser.add_argument("--num-display", action="store", default=6, type=int)
parser.add_argument("--dedup-frames", action="store", default=None, type=int)
parser.add_argument("--dedup-file", action="store_true")
parser.add_argument("--dedup-file", action="store", const=1, nargs='?')
parser.add_argument("--dedup-function", action="store_true")
args = parser.parse_args()
......@@ -20,7 +20,20 @@ if __name__ == "__main__":
key_func = lambda t: '\n'.join(t.split('\n')[-2 * args.dedup_frames:]) # last 4 stack frames
if args.dedup_file:
assert not key_func
key_func = lambda t: traceback_locations(t)[-1].split('"')[1]
def key_func(t):
locs = traceback_locations(t)
prev_f = None
n = int(args.dedup_file)
files = []
for l in reversed(locs):
f = l.split('"')[1]
if f == prev_f:
continue
prev_f = f
files.append(" " + f)
if len(files) == n:
break
return '\n'.join(reversed(files))
if args.dedup_function:
assert not key_func
def key_func(t):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment