I guess sre_parse.parse is slow

microbenchmarks/re_compile_bench.py

 import sre_compile
 import sre_constants
+import sre_parse
 import re
 
 IN = sre_constants.IN
@@ -7,26 +8,21 @@ IN = sre_constants.IN
 def _identity(x):
     return x
 
-_cache = {}
-DEBUG = 1
-def _compile(*key):
-    # print "re._compile", key
-    # internal: compile pattern
-    pattern, flags = key
-    bypass_cache = flags & DEBUG
-    if not bypass_cache:
-        cachekey = (type(key[0]),) + key
-        p = _cache.get(cachekey)
-        if p is not None:
-            # print "got from cache"
-            return p
-
-    p = (1, 0)
-    _cache[cachekey] = p
-    return p
-
 for i in xrange(1000000):
     # sre_compile._optimize_charset([('negate', None), ('literal', 34), ('literal', 92)], _identity)
     # sre_compile._compile([17, 4, 0, 3, 0, 29, 12, 0, 4294967295L, 15, 7, 26, 19, 34, 19, 92, 0, 1, 28, 0, 0, 4294967295L, 7, 6, 19, 92, 2, 18, 15, 13, 19, 34, 5, 7, 0, 19, 34, 19, 34, 1, 18, 2, 0, 29, 0, 0, 4294967295L], [(IN, [('negate', None), ('literal', 34), ('literal', 92)])], 0)
     # sre_compile._compile([17, 8, 3, 1, 1, 1, 1, 97, 0], [('literal', 97)], 0)
-    _compile("a", 0)
+    pass
+
+# This is the string tokenizer.PseudoToken:
+pattern = '[ \\f\\t]*((\\\\\\r?\\n|\\Z|#[^\\r\\n]*|([uUbB]?[rR]?\'\'\'|[uUbB]?[rR]?"""))|((\\d+[jJ]|((\\d+\\.\\d*|\\.\\d+)([eE][-+]?\\d+)?|\\d+[eE][-+]?\\d+)[jJ])|((\\d+\\.\\d*|\\.\\d+)([eE][-+]?\\d+)?|\\d+[eE][-+]?\\d+)|(0[xX][\\da-fA-F]+[lL]?|0[bB][01]+[lL]?|(0[oO][0-7]+)|(0[0-7]*)[lL]?|[1-9]\\d*[lL]?))|((\\*\\*=?|>>=?|<<=?|<>|!=|//=?|[+\\-*/%&|^=<>]=?|~)|[][(){}]|(\\r?\\n|[:;.,`@]))|([uUbB]?[rR]?\'[^\\n\'\\\\]*(?:\\\\.[^\\n\'\\\\]*)*(\'|\\\\\\r?\\n)|[uUbB]?[rR]?"[^\\n"\\\\]*(?:\\\\.[^\\n"\\\\]*)*("|\\\\\\r?\\n))|[a-zA-Z_]\\w*)'
+
+for i in xrange(100):
+    # re.compile checks if the pattern is in the cache, and then calls sre_compile:
+    sre_compile.compile(pattern, 0)
+
+for i in xrange(600):
+    p = sre_parse.Pattern()
+    p.flags = 0
+    p.str = pattern
+    sre_parse._parse_sub(sre_parse.Tokenizer(pattern), p, 0)

tools/process_stackdump.py

@@ -6,7 +6,7 @@ if __name__ == "__main__":
     parser.add_argument("tracebacks_file", action="store", default=None)
     parser.add_argument("--num-display", action="store", default=6, type=int)
     parser.add_argument("--dedup-frames", action="store", default=None, type=int)
-    parser.add_argument("--dedup-file", action="store_true")
+    parser.add_argument("--dedup-file", action="store", const=1, nargs='?')
     parser.add_argument("--dedup-function", action="store_true")
     args = parser.parse_args()
 
@@ -20,7 +20,20 @@ if __name__ == "__main__":
         key_func = lambda t: '\n'.join(t.split('\n')[-2 * args.dedup_frames:]) # last 4 stack frames
     if args.dedup_file:
         assert not key_func
-        key_func = lambda t: traceback_locations(t)[-1].split('"')[1]
+        def key_func(t):
+            locs = traceback_locations(t)
+            prev_f = None
+            n = int(args.dedup_file)
+            files = []
+            for l in reversed(locs):
+                f = l.split('"')[1]
+                if f == prev_f:
+                    continue
+                prev_f = f
+                files.append("  " + f)
+                if len(files) == n:
+                    break
+            return '\n'.join(reversed(files))
     if args.dedup_function:
         assert not key_func
         def key_func(t):