Introduce pre-splitting of code-stream in Code.py

Cython/Compiler/Buffer.py

@@ -1150,7 +1150,7 @@ static int __Pyx_GetBufferAndValidate(Py_buffer* buf, PyObject* obj, __Pyx_TypeI
     __Pyx_BufFmt_Init(&ctx, stack, dtype);
     if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail;
   }
-  if (buf->itemsize != dtype->size) {
+  if ((unsigned)buf->itemsize != dtype->size) {
     PyErr_Format(PyExc_ValueError,
       "Item size of buffer (%"PY_FORMAT_SIZE_T"d byte%s) does not match size of '%s' (%"PY_FORMAT_SIZE_T"d byte%s)",
       buf->itemsize, (buf->itemsize > 1) ? "s" : "",

Cython/Compiler/Code.py

@@ -213,6 +213,8 @@ class GlobalState(object):
     #                                  In time, hopefully the literals etc. will be
     #                                  supplied directly instead.
 
+    # parts            {string:CCodeWriter}
+
     
     # interned_strings
     # consts
@@ -226,7 +228,21 @@ class GlobalState(object):
 
     directives = {}
 
-    def __init__(self, rootwriter, emit_linenums=False):
+    code_layout = [
+        'h_code',
+        'before_global_var',
+        'global_var',
+        'after_global_var',
+        
+        'utility_proto',
+        'pystring_table',
+        'init_cached_builtins',
+        'init',
+        'utility_def',
+    ]
+    
+
+    def __init__(self, writer, emit_linenums=False):
         self.filename_table = {}
         self.filename_list = []
         self.input_file_contents = {}
@@ -235,6 +251,14 @@ class GlobalState(object):
         self.pystring_table_needed = False
         self.in_utility_code_generation = False
         self.emit_linenums = emit_linenums
+        self.parts = {}
+
+        assert writer.globalstate is None
+        writer.globalstate = self
+        for part in self.code_layout:
+            self.parts[part] = writer.insertion_point()#new_writer()
+        self.initwriters(writer)
+        
 
     def initwriters(self, rootwriter):
         self.utilprotowriter = rootwriter.new_writer()
@@ -261,12 +285,12 @@ class GlobalState(object):
         self.pystring_table.putln("static __Pyx_StringTabEntry %s[] = {" %
                 Naming.stringtab_cname)
 
+    def __getitem__(self, key):
+        return self.parts[key]
+
     #
     # Global constants, interned objects, etc.
     #
-    def insert_global_var_declarations_into(self, code):
-        code.insert(self.decls_writer)
-
     def close_global_decls(self):
         # This is called when it is known that no more global declarations will
         # declared (but can be called before or after insert_XXX).
@@ -310,7 +334,7 @@ class GlobalState(object):
         code.insert(self.cleanupwriter)
 
     def put_pyobject_decl(self, entry):
-        self.decls_writer.putln("static PyObject *%s;" % entry.cname)
+        self['global_var'].putln("static PyObject *%s;" % entry.cname)
 
     # The functions below are there in a transition phase only
     # and will be deprecated. They are called from Nodes.BlockNode.
@@ -329,16 +353,16 @@ class GlobalState(object):
 
     def add_const_definition(self, entry):
         if self.should_declare(entry.cname, entry):
-            self.decls_writer.put_var_declaration(entry, static = 1)
+            self['global_var'].put_var_declaration(entry, static = 1)
 
     def add_interned_string_decl(self, entry):
         if self.should_declare(entry.cname, entry):
-            self.decls_writer.put_var_declaration(entry, static = 1)
+            self['global_var'].put_var_declaration(entry, static = 1)
         self.add_py_string_decl(entry)
 
     def add_py_string_decl(self, entry):
         if self.should_declare(entry.pystring_cname, entry):
-            self.decls_writer.putln("static PyObject *%s;" % entry.pystring_cname)
+            self['global_var'].putln("static PyObject *%s;" % entry.pystring_cname)
             self.pystring_table_needed = True
             self.pystring_table.putln("{&%s, %s, sizeof(%s), %d, %d, %d}," % (
                 entry.pystring_cname,
@@ -503,6 +527,8 @@ class CCodeWriter(object):
     # globalstate      GlobalState     contains state global for a C file (input file info,
     #                                  utility code, declared constants etc.)
     # emit_linenums    boolean         whether or not to write #line pragmas 
+
+    globalstate = None
     
     def __init__(self, create_from=None, buffer=None, copy_formatting=False, emit_linenums=None):
         if buffer is None: buffer = StringIOTree()
@@ -515,12 +541,8 @@ class CCodeWriter(object):
         self.level = 0
         self.call_level = 0
         self.bol = 1
-        if create_from is None:
-            # Root CCodeWriter
-            self.globalstate = GlobalState(self, emit_linenums=emit_linenums)
-            self.globalstate.initwriters(self)
-            # ^^^ need seperate step because this will reference self.globalstate
-        else:
+
+        if create_from is not None:
             # Use same global state
             self.globalstate = create_from.globalstate
             # Clone formatting state
@@ -528,7 +550,7 @@ class CCodeWriter(object):
                 self.level = create_from.level
                 self.bol = create_from.bol
                 self.call_level = create_from.call_level
-        if emit_linenums is None:
+        if emit_linenums is None and self.globalstate:
             self.emit_linenums = self.globalstate.emit_linenums
         else:
             self.emit_linenums = emit_linenums
@@ -536,7 +558,8 @@ class CCodeWriter(object):
     def create_new(self, create_from, buffer, copy_formatting):
         # polymorphic constructor -- very slightly more versatile
         # than using __class__
-        return CCodeWriter(create_from, buffer, copy_formatting)
+        result = CCodeWriter(create_from, buffer, copy_formatting)
+        return result
 
     def copyto(self, f):
         self.buffer.copyto(f)

Cython/Compiler/ModuleNode.py

@@ -106,6 +106,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if h_types or h_vars or h_funcs or h_extension_types:
             result.h_file = replace_suffix(result.c_file, ".h")
             h_code = Code.CCodeWriter()
+            Code.GlobalState(h_code)
             if options.generate_pxi:
                 result.i_file = replace_suffix(result.c_file, ".pxi")
                 i_code = Code.PyrexCodeWriter(result.i_file)
@@ -167,6 +168,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if api_funcs or has_api_extension_types:
             result.api_file = replace_suffix(result.c_file, "_api.h")
             h_code = Code.CCodeWriter()
+            Code.GlobalState(h_code)
             name = self.api_name(env)
             guard = Naming.api_guard_prefix + name
             h_code.put_h_guard(guard)
@@ -243,18 +245,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     
     def generate_c_code(self, env, options, result):
         modules = self.referenced_modules
+
         if Options.annotate or options.annotate:
-            code = Annotate.AnnotationCCodeWriter()
+            emit_linenums = False
+            rootwriter = Annotate.AnnotationCCodeWriter()
         else:
-            code = Code.CCodeWriter(emit_linenums=options.emit_linenums)
-        h_code = code.insertion_point()
+            emit_linenums = options.emit_linenums
+            rootwriter = Code.CCodeWriter(emit_linenums=emit_linenums)
+        globalstate = Code.GlobalState(rootwriter, emit_linenums)
+        h_code = globalstate['h_code']
+        
         self.generate_module_preamble(env, modules, h_code)
 
-        code.globalstate.module_pos = self.pos
-        code.globalstate.directives = self.directives
+        globalstate.module_pos = self.pos
+        globalstate.directives = self.directives
 
-        code.globalstate.use_utility_code(refcount_utility_code)
+        globalstate.use_utility_code(refcount_utility_code)
 
+        code = globalstate['before_global_var']
         code.putln('#define __Pyx_MODULE_NAME "%s"' % self.full_module_name)
         code.putln("")
         code.putln("/* Implementation of %s */" % env.qualified_name)
@@ -263,8 +271,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_interned_string_decls(env, code)
         self.generate_py_string_decls(env, code)
 
-        code.globalstate.insert_global_var_declarations_into(code)
-        
+        code = globalstate['after_global_var']
+
         self.generate_cached_builtins_decls(env, code)
         self.body.generate_function_definitions(env, code)
         code.mark_pos(None)
@@ -285,15 +293,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_declarations_for_modules(env, modules, h_code)
         h_code.write('\n')
 
-        code.globalstate.close_global_decls()
+        globalstate.close_global_decls()
         
         f = open_new_file(result.c_file)
-        code.copyto(f)
+        rootwriter.copyto(f)
         f.close()
         result.c_file_generated = 1
         if Options.annotate or options.annotate:
-            self.annotate(code)
-            code.save_annotation(result.main_source_file, result.c_file)
+            self.annotate(rootwriter)
+            rootwriter.save_annotation(result.main_source_file, result.c_file)
     
     def find_referenced_modules(self, env, module_list, modules_seen):
         if env not in modules_seen:

Cython/Compiler/PyrexTypes.py

@@ -1061,6 +1061,7 @@ class CStructOrUnionType(CType):
         if self._convert_code is None:
             import Code
             code = Code.CCodeWriter()
+            Code.GlobalState(code)
             header = "static PyObject* %s(%s)" % (self.to_py_function, self.declaration_code('s'))
             code.putln("%s {" % header)
             code.putln("PyObject* res;")