C-ify some internal globals in Parsing.py, convert in-test tuples to faster sets

Cython/Compiler/Parsing.pxd

@@ -3,7 +3,7 @@
 cimport cython
 from Cython.Compiler.Scanning cimport PyrexScanner
 
-ctypedef object (*p_sub_expr_func)(object)
+ctypedef object (*p_sub_expr_func)(PyrexScanner obj)
 
 # entry points
 
@@ -17,26 +17,26 @@ cdef p_ident_list(PyrexScanner s)
 
 cdef tuple p_binop_operator(PyrexScanner s)
 cdef p_binop_expr(PyrexScanner s, ops, p_sub_expr_func p_sub_expr)
-cpdef p_lambdef(PyrexScanner s, bint allow_conditional=*)
+cdef p_lambdef(PyrexScanner s, bint allow_conditional=*)
 cdef p_lambdef_nocond(PyrexScanner s)
 cdef p_test(PyrexScanner s)
 cdef p_test_nocond(PyrexScanner s)
 cdef p_or_test(PyrexScanner s)
-cdef p_rassoc_binop_expr(PyrexScanner s, ops, p_subexpr)
-cpdef p_and_test(PyrexScanner s)
-cpdef p_not_test(PyrexScanner s)
+cdef p_rassoc_binop_expr(PyrexScanner s, ops, p_sub_expr_func p_subexpr)
+cdef p_and_test(PyrexScanner s)
+cdef p_not_test(PyrexScanner s)
 cdef p_comparison(PyrexScanner s)
 cdef p_test_or_starred_expr(PyrexScanner s)
 cdef p_starred_expr(PyrexScanner s)
 cdef p_cascaded_cmp(PyrexScanner s)
 cdef p_cmp_op(PyrexScanner s)
 cdef p_bit_expr(PyrexScanner s)
-cdef p_xor_expr(s)
-cdef p_and_expr(s)
-cdef p_shift_expr(s)
-cdef p_arith_expr(s)
-cdef p_term(s)
-cdef p_factor(s)
+cdef p_xor_expr(PyrexScanner s)
+cdef p_and_expr(PyrexScanner s)
+cdef p_shift_expr(PyrexScanner s)
+cdef p_arith_expr(PyrexScanner s)
+cdef p_term(PyrexScanner s)
+cdef p_factor(PyrexScanner s)
 cdef _p_factor(PyrexScanner s)
 cdef p_typecast(PyrexScanner s)
 cdef p_sizeof(PyrexScanner s)
@@ -45,7 +45,7 @@ cdef p_yield_statement(PyrexScanner s)
 cdef p_power(PyrexScanner s)
 cdef p_new_expr(PyrexScanner s)
 cdef p_trailer(PyrexScanner s, node1)
-cpdef p_call_parse_args(PyrexScanner s, bint allow_genexp = *)
+cdef p_call_parse_args(PyrexScanner s, bint allow_genexp = *)
 cdef p_call_build_packed_args(pos, positional_args, keyword_args, star_arg, starstar_arg)
 cdef p_call(PyrexScanner s, function)
 cdef p_index(PyrexScanner s, base)
@@ -71,7 +71,7 @@ cdef p_comp_for(PyrexScanner s, body)
 cdef p_comp_if(PyrexScanner s, body)
 cdef p_dict_or_set_maker(PyrexScanner s)
 cdef p_backquote_expr(PyrexScanner s)
-cpdef p_simple_expr_list(PyrexScanner s, expr=*)
+cdef p_simple_expr_list(PyrexScanner s, expr=*)
 cdef p_test_or_starred_expr_list(PyrexScanner s, expr=*)
 cdef p_testlist(PyrexScanner s)
 cdef p_testlist_star_expr(PyrexScanner s)
@@ -90,15 +90,15 @@ cdef p_expression_or_assignment(PyrexScanner s)
 cdef p_print_statement(PyrexScanner s)
 cdef p_exec_statement(PyrexScanner s)
 cdef p_del_statement(PyrexScanner s)
-cpdef p_pass_statement(PyrexScanner s, bint with_newline = *)
+cdef p_pass_statement(PyrexScanner s, bint with_newline = *)
 cdef p_break_statement(PyrexScanner s)
 cdef p_continue_statement(PyrexScanner s)
 cdef p_return_statement(PyrexScanner s)
 cdef p_raise_statement(PyrexScanner s)
 cdef p_import_statement(PyrexScanner s)
-cpdef p_from_import_statement(PyrexScanner s, bint first_statement = *)
+cdef p_from_import_statement(PyrexScanner s, bint first_statement = *)
 cdef p_imported_name(PyrexScanner s, bint is_cimport)
-cpdef p_dotted_name(PyrexScanner s, bint as_allowed)
+cdef p_dotted_name(PyrexScanner s, bint as_allowed)
 cdef p_as_name(PyrexScanner s)
 cdef p_assert_statement(PyrexScanner s)
 cdef p_if_statement(PyrexScanner s)
@@ -106,7 +106,7 @@ cdef p_if_clause(PyrexScanner s)
 cdef p_else_clause(PyrexScanner s)
 cdef p_while_statement(PyrexScanner s)
 cdef p_for_statement(PyrexScanner s)
-cpdef dict p_for_bounds(PyrexScanner s, bint allow_testlist = *)
+cdef dict p_for_bounds(PyrexScanner s, bint allow_testlist = *)
 cdef p_for_from_relation(PyrexScanner s)
 cdef p_for_from_step(PyrexScanner s)
 cdef p_target(PyrexScanner s, terminator)
@@ -117,21 +117,23 @@ cdef p_except_clause(PyrexScanner s)
 cdef p_include_statement(PyrexScanner s, ctx)
 cdef p_with_statement(PyrexScanner s)
 cdef p_with_items(PyrexScanner s)
-cpdef p_simple_statement(PyrexScanner s, bint first_statement = *)
-cpdef p_simple_statement_list(PyrexScanner s, ctx, bint first_statement = *)
+cdef p_with_template(PyrexScanner s)
+cdef p_simple_statement(PyrexScanner s, bint first_statement = *)
+cdef p_simple_statement_list(PyrexScanner s, ctx, bint first_statement = *)
 cdef p_compile_time_expr(PyrexScanner s)
 cdef p_DEF_statement(PyrexScanner s)
 cdef p_IF_statement(PyrexScanner s, ctx)
-cpdef p_statement(PyrexScanner s, ctx, bint first_statement = *)
-cpdef p_statement_list(PyrexScanner s, ctx, bint first_statement = *)
-cpdef p_suite(PyrexScanner s, ctx = *, bint with_doc = *, bint with_pseudo_doc = *)
+cdef p_statement(PyrexScanner s, ctx, bint first_statement = *)
+cdef p_statement_list(PyrexScanner s, ctx, bint first_statement = *)
+cdef p_suite(PyrexScanner s, ctx = *, bint with_doc = *, bint with_pseudo_doc = *)
 cdef p_positional_and_keyword_args(PyrexScanner s, end_sy_set, templates = *)
 
 cpdef p_c_base_type(PyrexScanner s, bint self_flag = *, bint nonempty = *, templates = *)
 cdef p_calling_convention(PyrexScanner s)
 cdef p_c_complex_base_type(PyrexScanner s)
-cpdef p_c_simple_base_type(PyrexScanner s, bint self_flag, bint nonempty, templates = *)
+cdef p_c_simple_base_type(PyrexScanner s, bint self_flag, bint nonempty, templates = *)
 cdef p_buffer_or_template(PyrexScanner s, base_type_node, templates)
+cdef p_bracketed_base_type(PyrexScanner s, base_type_node, nonempty, empty)
 cdef is_memoryviewslice_access(PyrexScanner s)
 cdef p_memoryviewslice_access(PyrexScanner s, base_type_node)
 cdef bint looking_at_name(PyrexScanner s) except -2
@@ -154,7 +156,7 @@ cdef p_exception_value_clause(PyrexScanner s)
 cpdef p_c_arg_list(PyrexScanner s, ctx = *, bint in_pyfunc = *, bint cmethod_flag = *,
                    bint nonempty_declarators = *, bint kw_only = *, bint annotated = *)
 cdef p_optional_ellipsis(PyrexScanner s)
-cpdef p_c_arg_decl(PyrexScanner s, ctx, in_pyfunc, bint cmethod_flag = *, bint nonempty = *, bint kw_only = *, bint annotated = *)
+cdef p_c_arg_decl(PyrexScanner s, ctx, in_pyfunc, bint cmethod_flag = *, bint nonempty = *, bint kw_only = *, bint annotated = *)
 cdef p_api(PyrexScanner s)
 cdef p_cdef_statement(PyrexScanner s, ctx)
 cdef p_cdef_block(PyrexScanner s, ctx)
@@ -164,14 +166,15 @@ cdef p_c_enum_line(PyrexScanner s, ctx, list items)
 cdef p_c_enum_item(PyrexScanner s, ctx, list items)
 cdef p_c_struct_or_union_definition(PyrexScanner s, pos, ctx)
 cdef p_fused_definition(PyrexScanner s, pos, ctx)
+cdef p_struct_enum(PyrexScanner s, pos, ctx)
 cdef p_visibility(PyrexScanner s, prev_visibility)
 cdef p_c_modifiers(PyrexScanner s)
 cdef p_c_func_or_var_declaration(PyrexScanner s, pos, ctx)
 cdef p_ctypedef_statement(PyrexScanner s, ctx)
 cdef p_decorators(PyrexScanner s)
 cdef p_def_statement(PyrexScanner s, list decorators = *)
-cpdef p_varargslist(PyrexScanner s, terminator=*, bint annotated = *)
-cpdef p_py_arg_decl(PyrexScanner s, bint annotated = *)
+cdef p_varargslist(PyrexScanner s, terminator=*, bint annotated = *)
+cdef p_py_arg_decl(PyrexScanner s, bint annotated = *)
 cdef p_class_statement(PyrexScanner s, decorators)
 cdef p_c_class_definition(PyrexScanner s, pos,  ctx)
 cdef p_c_class_options(PyrexScanner s)

Cython/Compiler/Parsing.py

@@ -6,9 +6,11 @@
 # This should be done automatically
 import cython
 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object,
-               StringEncoding=object, lookup_unicodechar=object, re=object,
+               BytesLiteral=object, StringEncoding=object,
+               FileSourceDescriptor=object, lookup_unicodechar=object,
                Future=object, Options=object, error=object, warning=object,
-               Builtin=object)
+               Builtin=object, ModuleNode=object, Utils=object,
+               re=object, _unicode=object, _bytes=object)
 
 import re
 from unicodedata import lookup as lookup_unicodechar
@@ -235,10 +237,10 @@ def p_cmp_op(s):
         op = '!='
     return op
 
-comparison_ops = (
+comparison_ops = cython.declare(set, set([
     '<', '>', '==', '>=', '<=', '<>', '!=',
     'in', 'is', 'not'
-)
+]))
 
 #expr: xor_expr ('|' xor_expr)*
 
@@ -1083,7 +1085,8 @@ def p_genexp(s, expr):
         expr.pos, expr = ExprNodes.YieldExprNode(expr.pos, arg=expr)))
     return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop)
 
-expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')
+expr_terminators = cython.declare(set, set([
+    ')', ']', '}', ':', '=', 'NEWLINE']))
 
 #-------------------------------------------------------
 #
@@ -1392,7 +1395,8 @@ def p_from_import_statement(s, first_statement = 0):
                 name_list = import_list),
             items = items)
 
-imported_name_kinds = ('class', 'struct', 'union')
+imported_name_kinds = cython.declare(
+    set, set(['class', 'struct', 'union']))
 
 def p_imported_name(s, is_cimport):
     pos = s.position()
@@ -1435,7 +1439,7 @@ def p_assert_statement(s):
         value = None
     return Nodes.AssertStatNode(pos, cond = cond, value = value)
 
-statement_terminators = (';', 'NEWLINE', 'EOF')
+statement_terminators = cython.declare(set, set([';', 'NEWLINE', 'EOF']))
 
 def p_if_statement(s):
     # s.sy == 'if'
@@ -1543,7 +1547,7 @@ def p_for_from_step(s):
     else:
         return None
 
-inequality_relations = ('<', '<=', '>', '>=')
+inequality_relations = cython.declare(set, set(['<', '<=', '>', '>=']))
 
 def p_target(s, terminator):
     pos = s.position()
@@ -2017,7 +2021,8 @@ def p_calling_convention(s):
     else:
         return ""
 
-calling_convention_words = ("__stdcall", "__cdecl", "__fastcall")
+calling_convention_words = cython.declare(
+    set, set(["__stdcall", "__cdecl", "__fastcall"]))
 
 def p_c_complex_base_type(s):
     # s.sy == '('
@@ -2255,9 +2260,10 @@ def looking_at_call(s):
         s.start_line, s.start_col = position
     return result
 
-basic_c_type_names = ("void", "char", "int", "float", "double", "bint")
+basic_c_type_names = cython.declare(
+    set, set(["void", "char", "int", "float", "double", "bint"]))
 
-special_basic_c_types = {
+special_basic_c_types = cython.declare(dict, {
     # name : (signed, longness)
     "Py_UNICODE" : (0, 0),
     "Py_UCS4"    : (0, 0),
@@ -2265,14 +2271,19 @@ special_basic_c_types = {
     "ssize_t"    : (2, 0),
     "size_t"     : (0, 0),
     "ptrdiff_t"  : (2, 0),
-}
+})
 
-sign_and_longness_words = ("short", "long", "signed", "unsigned")
+sign_and_longness_words = cython.declare(
+    set, set(["short", "long", "signed", "unsigned"]))
 
-base_type_start_words = \
-    basic_c_type_names + sign_and_longness_words + tuple(special_basic_c_types)
+base_type_start_words = cython.declare(
+    set,
+    basic_c_type_names
+    | sign_and_longness_words
+    | set(special_basic_c_types))
 
-struct_enum_union = ("struct", "union", "enum", "packed")
+struct_enum_union = cython.declare(
+    set, set(["struct", "union", "enum", "packed"]))
 
 def p_sign_and_longness(s):
     signed = 1
@@ -2357,12 +2368,12 @@ def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):
         exception_value = exc_val, exception_check = exc_check,
         nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)
 
-supported_overloaded_operators = set([
+supported_overloaded_operators = cython.declare(set, set([
     '+', '-', '*', '/', '%',
     '++', '--', '~', '|', '&', '^', '<<', '>>', ',',
     '==', '!=', '>=', '>', '<=', '<',
     '[]', '()', '!',
-])
+]))
 
 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
                           assignable, nonempty):
@@ -2480,7 +2491,7 @@ def p_exception_value_clause(s):
             exc_val = p_test(s)
     return exc_val, exc_check
 
-c_arg_list_terminators = ('*', '**', '.', ')')
+c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')']))
 
 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
                  nonempty_declarators = 0, kw_only = 0, annotated = 1):
@@ -3066,17 +3077,18 @@ def p_code(s, level=None, ctx=Ctx):
             repr(s.sy), repr(s.systring)))
     return body
 
-COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython\s*:\s*((\w|[.])+\s*=.*)$")
+_match_compiler_directive_comment = cython.declare(object, re.compile(
+    r"^#\s*cython\s*:\s*((\w|[.])+\s*=.*)$").match)
 
 def p_compiler_directive_comments(s):
     result = {}
     while s.sy == 'commentline':
-        m = COMPILER_DIRECTIVE_COMMENT_RE.match(s.systring)
+        m = _match_compiler_directive_comment(s.systring)
         if m:
             directives = m.group(1).strip()
             try:
-                result.update( Options.parse_directive_list(
-                    directives, ignore_unknown=True) )
+                result.update(Options.parse_directive_list(
+                    directives, ignore_unknown=True))
             except ValueError, e:
                 s.error(e.args[0], fatal=False)
         s.next()