Generating pxd code, but problem with interned strings etc. (very unstable!)

Cython/Compiler/Main.py

@@ -22,6 +22,7 @@ from Scanning import PyrexScanner, FileSourceDescriptor
 from Errors import PyrexError, CompileError, error
 from Symtab import BuiltinScope, ModuleScope
 from Cython import Utils
+from Cython.Utils import open_new_file, replace_suffix
 
 module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$")
 
@@ -32,6 +33,20 @@ def dumptree(t):
     print t.dump()
     return t
 
+class CompilationData:
+    #  Bundles the information that is passed from transform to transform.
+    #  (For now, this is only)
+
+    #  While Context contains every pxd ever loaded, path information etc.,
+    #  this only contains the data related to a single compilation pass
+    #
+    #  pyx                   ModuleNode              Main code tree of this compilation.
+    #  pxds                  {string : ModuleNode}   Trees for the pxds used in the pyx.
+    #  codewriter            CCodeWriter             Where to output final code.
+    #  options               CompilationOptions
+    #  result                CompilationResult
+    pass
+
 class Context:
     #  This class encapsulates the context needed for compiling
     #  one or more Cython implementation files along with their
@@ -50,6 +65,8 @@ class Context:
         self.include_directories = include_directories
         self.future_directives = set()
 
+        self.pxds = {} # full name -> node tree
+
         standard_include_path = os.path.abspath(
             os.path.join(os.path.dirname(__file__), '..', 'Includes'))
         self.include_directories = include_directories + [standard_include_path]
@@ -87,9 +104,26 @@ class Context:
             ]
 
     def create_pyx_pipeline(self, options, result):
-        return [create_parse(self)] + self.create_pipeline(pxd=False) + [
-            create_generate_code(self, options, result)
-            ]
+        def generate_pyx_code(module_node):
+            module_node.process_implementation(options, result)
+            result.compilation_source = module_node.compilation_source
+            return result
+
+        def inject_pxd_code(module_node):
+            from Nodes import CCommentNode
+            from textwrap import dedent
+            stats = module_node.body.stats
+            for name, (statlistnode, scope) in self.pxds.iteritems():
+                #stats.append(CCommentNode('Code from cimported "%s"' % name, header=True))
+                stats.append(statlistnode)
+            return module_node
+
+        return ([
+                create_parse(self)
+            ] + self.create_pipeline(pxd=False) + [
+                inject_pxd_code,
+                generate_pyx_code,
+            ])
 
     def create_pxd_pipeline(self, scope, module_name):
         def parse_pxd(source_desc):
@@ -98,7 +132,14 @@ class Context:
             tree.scope = scope
             tree.is_pxd = True
             return tree
-        return [parse_pxd] + self.create_pipeline(pxd=True)
+
+        from CodeGeneration import ExtractPxdCode
+
+        # The pxd pipeline ends up with a CCodeWriter containing the
+        # code of the pxd, as well as a pxd scope.
+        return [parse_pxd] + self.create_pipeline(pxd=True) + [
+            ExtractPxdCode(self)
+            ]
 
     def process_pxd(self, source_desc, scope, module_name):
         pipeline = self.create_pxd_pipeline(scope, module_name)
@@ -171,7 +212,8 @@ class Context:
                     if debug_find_module:
                         print("Context.find_module: Parsing %s" % pxd_pathname)
                     source_desc = FileSourceDescriptor(pxd_pathname)
-                    self.process_pxd(source_desc, scope, module_name)
+                    errors_occured, (pxd_codenodes, pxd_scope) = self.process_pxd(source_desc, scope, module_name)
+                    self.pxds[module_name] = (pxd_codenodes, pxd_scope)
                 except CompileError:
                     pass
         return scope
@@ -406,14 +448,6 @@ def create_parse(context):
         return tree
     return parse
 
-def create_generate_code(context, options, result):
-    def generate_code(module_node):
-        scope = module_node.scope
-        module_node.process_implementation(options, result)
-        result.compilation_source = module_node.compilation_source
-        return result
-    return generate_code
-
 def create_default_resultobj(compilation_source, options):
     result = CompilationResult()
     result.main_source_file = compilation_source.source_desc.filename

Cython/Compiler/Nodes.py

@@ -3885,6 +3885,34 @@ class FromImportStatNode(StatNode):
             target.generate_assignment_code(self.item, code)
         self.module.generate_disposal_code(code)
 
+
+class CForeignScopeNode(StatNode):
+    """
+    Used for wrapping FuncDefNodes from pxds so that they generate their
+    code in the right scope.
+    """
+    pass
+
+class CCommentNode(StatNode):
+    # not working!
+    def __init__(self, comment, header=False):
+        self.pos 
+        assert '/' not in comment # todo: better escaping
+        if header:
+            from textwrap import dedent
+            comment = dedent("""
+            /*
+             *   %s
+             */
+            """) % comment
+        else:
+            comment = "/* %s */" % comment
+        self.comment = comment
+      
+    def generate_execution_code(self, code):
+        code.putln(self.header)
+    
+
 #------------------------------------------------------------------------------------
 #
 #  Runtime support code

Cython/Compiler/ParseTreeTransforms.py

@@ -217,6 +217,12 @@ class PxdPostParse(CythonTransform):
     """
     Basic interpretation/validity checking that should only be
     done on pxd trees.
+
+    A lot of this checking currently happens in the parser; but
+    what is listed below happens here.
+
+    - "def" functions are let through only if they fill the
+    getbuffer/releasebuffer slots
     """
     ERR_FUNCDEF_NOT_ALLOWED = 'function definition not allowed here'
 
@@ -240,7 +246,6 @@ class PxdPostParse(CythonTransform):
             and node.name in ('__getbuffer__', '__releasebuffer__')):
             ok = True
 
-
         if not ok:
             self.context.nonfatal_error(PostParseError(node.pos,
                 self.ERR_FUNCDEF_NOT_ALLOWED))