Add option to use formal grammar.

Cython/Compiler/Main.py

@@ -304,6 +304,13 @@ class Context(object):
                 s = PyrexScanner(f, source_desc, source_encoding = f.encoding,
                                  scope = scope, context = self)
                 tree = Parsing.p_module(s, pxd, full_module_name)
+                if Options.formal_grammar:
+                    try:
+                        from ..Parser import ConcreteSyntaxTree
+                    except ImportError:
+                        raise RuntimeError(
+                            "Formal grammer can only be used with compiled Cython with an available pgen.")
+                    ConcreteSyntaxTree.p_module(source_filename)
             finally:
                 f.close()
         except UnicodeDecodeError, e:

Cython/Compiler/Options.py

@@ -80,6 +80,9 @@ closure_freelist_size = 8
 # Should tp_clear() set object fields to None instead of clearing them to NULL?
 clear_to_none = True
 
+# Should we try parsing files with the formal grammar (experimental)?
+formal_grammar = True
+
 
 # Declare compiler directives
 directive_defaults = {

Cython/Parser/ConcreteSyntaxTree.pyx

+cdef extern from "graminit.c":
+    pass
+
+def p_module(source):
+    print "Using formal grammar to parse", source

setup.py

@@ -4,6 +4,7 @@ try:
 except ImportError:
     from distutils.core import setup, Extension
 import os
+import subprocess
 import sys
 
 try:
@@ -28,7 +29,7 @@ from distutils.command.sdist import sdist as sdist_orig
 class sdist(sdist_orig):
     def run(self):
         self.force_manifest = 1
-        if (sys.platform != "win32" and 
+        if (sys.platform != "win32" and
             os.path.isdir('.git')):
             assert os.system("git rev-parse --verify HEAD > .gitrev") == 0
         sdist_orig.run(self)
@@ -119,6 +120,30 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan
             "Cython.Compiler.Optimize",
             ])
 
+    from distutils.spawn import find_executable
+    from distutils.sysconfig import get_python_inc
+    pgen = find_executable(
+        'pgen', os.pathsep.join([os.environ['PATH'], os.path.join(get_python_inc(), '..', 'Parser')]))
+    if not pgen:
+        print "Unable to find pgen, not compiling formal grammar."
+    else:
+        parser_dir = os.path.join(os.path.dirname(__file__), 'Cython', 'Parser')
+        print ' '.join([
+            pgen,
+            os.path.join(get_python_inc(), '..', 'Grammar', 'Grammar'),
+            os.path.join(parser_dir, 'graminit.h'),
+            os.path.join(parser_dir, 'graminit.c'),
+            ])
+        subprocess.check_call([
+            pgen,
+            os.path.join(get_python_inc(), '..', 'Grammar', 'Grammar'),
+            os.path.join(parser_dir, 'graminit.h'),
+            os.path.join(parser_dir, 'graminit.c'),
+            ])
+        compiled_modules.extend([
+                "Cython.Parser.ConcreteSyntaxTree",
+            ])
+
     defines = []
     if cython_with_refnanny:
         defines.append(('CYTHON_REFNANNY', '1'))