Parse files with standard Python grammar.

Cython/Compiler/Main.py

@@ -304,7 +304,7 @@ class Context(object):
                 s = PyrexScanner(f, source_desc, source_encoding = f.encoding,
                                  scope = scope, context = self)
                 tree = Parsing.p_module(s, pxd, full_module_name)
-                if Options.formal_grammar:
+                if self.options.formal_grammar:
                     try:
                         from ..Parser import ConcreteSyntaxTree
                     except ImportError:
@@ -482,6 +482,7 @@ class CompilationOptions(object):
     compiler_directives  dict      Overrides for pragma options (see Options.py)
     evaluate_tree_assertions boolean  Test support: evaluate parse tree assertions
     language_level    integer   The Python language level: 2 or 3
+    formal_grammar    boolean  Parse the file with the formal grammar
 
     cplus             boolean   Compile as c++ code
     """
@@ -511,6 +512,8 @@ class CompilationOptions(object):
         options['compiler_directives'] = directives
         if 'language_level' in directives and 'language_level' not in kw:
             options['language_level'] = int(directives['language_level'])
+        if 'formal_grammar' in directives and 'formal_grammar' not in kw:
+            options['formal_grammar'] = directives['formal_grammar']
         if 'cache' in options:
             if options['cache'] is True:
                 options['cache'] = os.path.expanduser("~/.cycache")
@@ -689,6 +692,7 @@ default_options = dict(
     relative_path_in_code_position_comments = True,
     c_line_in_traceback = True,
     language_level = 2,
+    formal_grammar = False,
     gdb_debug = False,
     compile_time_env = None,
     common_utility_include_dir = None,

Cython/Compiler/Options.py

@@ -80,9 +80,6 @@ closure_freelist_size = 8
 # Should tp_clear() set object fields to None instead of clearing them to NULL?
 clear_to_none = True
 
-# Should we try parsing files with the formal grammar (experimental)?
-formal_grammar = True
-
 
 # Declare compiler directives
 directive_defaults = {
@@ -150,6 +147,8 @@ directive_defaults = {
 # experimental, subject to change
     'binding': None,
     'freelist': 0,
+
+    'formal_grammar': False,
 }
 
 # Extra warning directives

Cython/Parser/ConcreteSyntaxTree.pyx

 cdef extern from "graminit.c":
-    pass
+    ctypedef struct grammar:
+        pass
+    cdef grammar _PyParser_Grammar
+    cdef int Py_file_input
 
-def p_module(source):
-    print "Using formal grammar to parse", source
+cdef extern from "node.h":
+    ctypedef struct node
+    void PyNode_Free(node*)
+    int NCH(node*)
+    node* CHILD(node*, int)
+    node* RCHILD(node*, int)
+    short TYPE(node*)
+    char* STR(node*)
+
+cdef extern from "parsetok.h":
+    ctypedef struct perrdetail:
+        pass
+    cdef void PyParser_SetError(perrdetail *err) except *
+    cdef node * PyParser_ParseStringFlagsFilenameEx(
+        const char * s,
+        const char * filename,
+        grammar * g,
+        int start,
+        perrdetail * err_ret,
+        int * flags)
+
+import distutils.sysconfig
+import os
+
+def extract_names(path):
+    # All parse tree types are #defined in these files as ints.
+    type_names = {}
+    for line in open(path):
+        if line.startswith('#define'):
+            try:
+                _, name, value = line.strip().split()
+                type_names[int(value)] = name
+            except:
+                pass
+    return type_names
+
+cdef dict type_names = {}
+
+cdef print_tree(node* n, indent=""):
+    if not type_names:
+        type_names.update(extract_names(
+            os.path.join(distutils.sysconfig.get_python_inc(), 'token.h')))
+        type_names.update(extract_names(
+            os.path.join(os.path.dirname(__file__), 'graminit.h')))
+
+    print indent, type_names.get(TYPE(n), 'unknown'), <object>STR(n) if NCH(n) == 0 else NCH(n)
+    indent += "  "
+    for i in range(NCH(n)):
+        print_tree(CHILD(n, i), indent)
+
+def p_module(path):
+    cdef perrdetail err
+    cdef int flags
+    cdef node* n
+    source = open(path).read()
+    n = PyParser_ParseStringFlagsFilenameEx(
+        source,
+        path,
+        &_PyParser_Grammar,
+        Py_file_input,
+        &err,
+        &flags)
+    if n:
+        print_tree(n)
+        PyNode_Free(n)
+    else:
+        PyParser_SetError(&err)