Utility code code stream refactor

Cython/Compiler/Buffer.py

@@ -413,7 +413,11 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives, pos,
             params.append(s.cname)
         
     # Make sure the utility code is available
-    code.globalstate.use_code_from(funcgen, name=funcname, nd=nd)
+    if funcname not in code.globalstate.utility_codes:
+        code.globalstate.utility_codes.add(funcname)
+        protocode = code.globalstate['utility_code_proto']
+        defcode = code.globalstate['utility_code_def']
+        funcgen(protocode, defcode, name=funcname, nd=nd)
 
     ptr_type = entry.type.buffer_ptr_type
     ptrcode = "%s(%s, %s.buf, %s)" % (funcname,
@@ -581,82 +585,76 @@ def mangle_dtype_name(dtype):
         return prefix + dtype.declaration_code("").replace(" ", "_")
 
 def get_type_information_cname(code, dtype, maxdepth=None):
-    # Output the __Pyx_TypeInfo type information for the given dtype if needed,
+    # Output the run-time type information (__Pyx_TypeInfo) for given dtype,
     # and return the name of the type info struct.
-    namesuffix = mangle_dtype_name(dtype)
-    name = "__Pyx_TypeInfo_%s" % namesuffix
-    structinfo_name = "__Pyx_StructFields_%s" % namesuffix
-
-    # It's critical that walking the type info doesn't use more stack
-    # depth than dtype.struct_nesting_depth() returns, so use an assertion for this
-    if maxdepth is None: maxdepth = dtype.struct_nesting_depth()
-    code.globalstate.use_code_from(type_information_code, name,
-                                   structinfo_name=structinfo_name,
-                                   dtype=dtype, maxdepth=maxdepth)
-    return name
-
-def type_information_code(proto, impl, name, structinfo_name, dtype, maxdepth):
-    # Output the run-time type information (__Pyx_TypeInfo) for given dtype.
-    # Use through get_type_information_cname
     #
     # Structs with two floats of the same size are encoded as complex numbers.
     # One can seperate between complex numbers declared as struct or with native
     # encoding by inspecting to see if the fields field of the type is
     # filled in.
+    namesuffix = mangle_dtype_name(dtype)
+    name = "__Pyx_TypeInfo_%s" % namesuffix
+    structinfo_name = "__Pyx_StructFields_%s" % namesuffix
 
-    if dtype.is_error: return
-    complex_possible = dtype.is_struct_or_union and dtype.can_be_complex()
-
-    code = proto.globalstate['typeinfo']
+    if dtype.is_error: return "<error>"
 
+    # It's critical that walking the type info doesn't use more stack
+    # depth than dtype.struct_nesting_depth() returns, so use an assertion for this
+    if maxdepth is None: maxdepth = dtype.struct_nesting_depth()
     if maxdepth <= 0:
         assert False
 
-    declcode = dtype.declaration_code("")
-    if dtype.is_simple_buffer_dtype():
-        structinfo_name = "NULL"
-    elif dtype.is_struct:
-        fields = dtype.scope.var_entries
-        # Must pre-call all used types in order not to recurse utility code
-        # writing.
-        assert len(fields) > 0
-        types = [get_type_information_cname(proto, f.type, maxdepth - 1)
-                 for f in fields]
-        code.putln("static __Pyx_StructField %s[] = {" % structinfo_name, safe=True)
-        for f, typeinfo in zip(fields, types):
-            code.putln('  {&%s, "%s", offsetof(%s, %s)},' %
-                       (typeinfo, f.name, dtype.declaration_code(""), f.cname), safe=True)
-        code.putln('  {NULL, NULL, 0}', safe=True)
-        code.putln("};", safe=True)
-    else:
-        assert False
+    if name not in code.globalstate.utility_codes:
+        code.globalstate.utility_codes.add(name)
+        typecode = code.globalstate['typeinfo']
+        
+        complex_possible = dtype.is_struct_or_union and dtype.can_be_complex()
+        
+        declcode = dtype.declaration_code("")
+        if dtype.is_simple_buffer_dtype():
+            structinfo_name = "NULL"
+        elif dtype.is_struct:
+            fields = dtype.scope.var_entries
+            # Must pre-call all used types in order not to recurse utility code
+            # writing.
+            assert len(fields) > 0
+            types = [get_type_information_cname(proto, f.type, maxdepth - 1)
+                     for f in fields]
+            typecode.putln("static __Pyx_StructField %s[] = {" % structinfo_name, safe=True)
+            for f, typeinfo in zip(fields, types):
+                typecode.putln('  {&%s, "%s", offsetof(%s, %s)},' %
+                           (typeinfo, f.name, dtype.declaration_code(""), f.cname), safe=True)
+            typecode.putln('  {NULL, NULL, 0}', safe=True)
+            typecode.putln("};", safe=True)
+        else:
+            assert False
             
-    rep = str(dtype)
-    if dtype.is_int:
-        if dtype.signed == 0:
-            typegroup = 'U'
+        rep = str(dtype)
+        if dtype.is_int:
+            if dtype.signed == 0:
+                typegroup = 'U'
+            else:
+                typegroup = 'I'
+        elif complex_possible:
+            typegroup = 'C'
+        elif dtype.is_float:
+            typegroup = 'R'
+        elif dtype.is_struct:
+            typegroup = 'S'
+        elif dtype.is_pyobject:
+            typegroup = 'O'
         else:
-            typegroup = 'I'
-    elif complex_possible:
-        typegroup = 'C'
-    elif dtype.is_float:
-        typegroup = 'R'
-    elif dtype.is_struct:
-        typegroup = 'S'
-    elif dtype.is_pyobject:
-        typegroup = 'O'
-    else:
-        print dtype
-        assert False
-
-    code.putln(('static __Pyx_TypeInfo %s = { "%s", %s, sizeof(%s), \'%s\' };'
-                ) % (name,
-                     rep,
-                     structinfo_name,
-                     declcode,
-                     typegroup,
-                     ), safe=True)
+            print dtype
+            assert False
 
+        typecode.putln(('static __Pyx_TypeInfo %s = { "%s", %s, sizeof(%s), \'%s\' };'
+                        ) % (name,
+                             rep,
+                             structinfo_name,
+                             declcode,
+                             typegroup,
+                        ), safe=True)
+    return name
 
 
 # Utility function to set the right exception

Cython/Compiler/Code.py

@@ -7,6 +7,7 @@ import Naming
 import Options
 from Cython.Utils import open_new_file, open_source_file
 from PyrexTypes import py_object_type, typecast
+import PyrexTypes
 from TypeSlots import method_coexist
 from Scanning import SourceDescriptor
 from Cython.StringIOTree import StringIOTree
@@ -203,9 +204,7 @@ class GlobalState(object):
     #                                  to create this output C code.  This is
     #                                  used to annotate the comments.
     #
-    # used_utility_code set(string|int) Ids of used utility code (to avoid reinsertion)
-    # utilprotowriter CCodeWriter
-    # utildefwriter   CCodeWriter
+    # utility_codes   set                IDs of used utility code (to avoid reinsertion)
     #
     # declared_cnames  {string:Entry}  used in a transition phase to merge pxd-declared
     #                                  constants etc. into the pyx-declared ones (i.e,
@@ -230,12 +229,14 @@ class GlobalState(object):
 
     code_layout = [
         'h_code',
+        'utility_code_proto',
         'type_declarations',
         'module_declarations',
         'typeinfo',
         'before_global_var',
         'global_var',
         'all_the_rest',
+        'utility_code_def'
     ]
     
 
@@ -243,7 +244,7 @@ class GlobalState(object):
         self.filename_table = {}
         self.filename_list = []
         self.input_file_contents = {}
-        self.used_utility_code = set()
+        self.utility_codes = set()
         self.declared_cnames = {}
         self.pystring_table_needed = False
         self.in_utility_code_generation = False
@@ -254,12 +255,11 @@ class GlobalState(object):
         writer.globalstate = self
         for part in self.code_layout:
             self.parts[part] = writer.insertion_point()#new_writer()
-        self.initwriters(writer)
+
+        self.init_writers(writer)
         
 
-    def initwriters(self, rootwriter):
-        self.utilprotowriter = rootwriter.new_writer()
-        self.utildefwriter = rootwriter.new_writer()
+    def init_writers(self, rootwriter):
         self.decls_writer = rootwriter.new_writer()
         self.pystring_table = rootwriter.new_writer()
         self.init_cached_builtins_writer = rootwriter.new_writer()
@@ -282,6 +282,31 @@ class GlobalState(object):
         self.pystring_table.putln("static __Pyx_StringTabEntry %s[] = {" %
                 Naming.stringtab_cname)
 
+
+        #
+        # utility_code_def
+        #
+        code = self.parts['utility_code_def']
+        if self.emit_linenums:
+            code.write('\n#line 1 "cython_utility"\n')
+        code.putln("")
+        code.putln("/* Runtime support code */")
+        code.putln("")
+        code.putln("static void %s(void) {" % Naming.fileinit_cname)
+        code.putln("%s = %s;" % 
+            (Naming.filetable_cname, Naming.filenames_cname))
+        code.putln("}")
+
+    def finalize_writers(self):
+        self.close_global_decls()
+
+        #
+        # utility_code_def
+        #
+        code = self.parts['utility_code_def']
+        code.put(PyrexTypes.type_conversion_functions)
+        code.putln("")
+
     def __getitem__(self, key):
         return self.parts[key]
 
@@ -439,55 +464,18 @@ class GlobalState(object):
         code twice. Otherwise, id(codetup) is used as such an identifier.
         """
         if name is None: name = id(utility_code)
-        if self.check_utility_code_needed_and_register(name):
+        if name not in self.utility_codes:
+            self.utility_codes.add(name)
             if utility_code.requires:
                 for dependency in utility_code.requires:
                     self.use_utility_code(dependency)
             if utility_code.proto:
-                self.utilprotowriter.put(utility_code.proto)
+                self.parts['utility_code_proto'].put(utility_code.proto)
             if utility_code.impl:
-                self.utildefwriter.put(utility_code.impl)
+                self.parts['utility_code_def'].put(utility_code.impl)
             utility_code.write_init_code(self.initwriter, self.module_pos)
             utility_code.write_cleanup_code(self.cleanupwriter, self.module_pos)
 
-    def has_code(self, name):
-        return name in self.used_utility_code
-
-    def use_code_from(self, func, name, *args, **kw):
-        """
-        Requests that the utility code that func can generate is used in the C
-        file. func is called like this:
-
-        func(proto, definition, name, *args, **kw)
-
-        where proto and definition are two CCodeWriter instances; the
-        former should have the prototype written to it and the other the definition.
-        
-        The call might happen at some later point (if compiling multiple modules
-        into a cache for instance), and will only happen once per utility code.
-
-        name is used to identify the utility code, so that it isn't regenerated
-        when the same code is requested again.
-        """
-        if self.check_utility_code_needed_and_register(name):
-            func(self.utilprotowriter, self.utildefwriter,
-                 name, *args, **kw)
-
-    def check_utility_code_needed_and_register(self, name):
-        if name in self.used_utility_code:
-            return False
-        else:
-            self.used_utility_code.add(name)
-            return True
-
-    def put_utility_code_protos(self, writer):
-        writer.insert(self.utilprotowriter)
-
-    def put_utility_code_defs(self, writer):
-        if self.emit_linenums:
-            writer.write('\n#line 1 "cython_utility"\n')
-        writer.insert(self.utildefwriter)
-
 
 def funccontext_property(name):
     def get(self):

Cython/Compiler/ModuleNode.py

@@ -288,12 +288,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if Options.embed:
             self.generate_main_method(env, code)
         self.generate_filename_table(code)
-        self.generate_utility_functions(env, code, h_code)
-
+        
         self.generate_declarations_for_modules(env, modules, globalstate)
         h_code.write('\n')
 
-        globalstate.close_global_decls()
+        for codetup, name in env.utility_code_list:
+            globalstate.use_utility_code(codetup, name)
+        globalstate.finalize_writers()
         
         f = open_new_file(result.c_file)
         rootwriter.copyto(f)
@@ -2061,22 +2062,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 "%s = &%s;" % (
                     type.typeptr_cname, type.typeobj_cname))
     
-    def generate_utility_functions(self, env, code, h_code):
-        for codetup, name in env.utility_code_list:
-            code.globalstate.use_utility_code(codetup, name)
-       
-        code.globalstate.put_utility_code_protos(h_code)
-        code.putln("")
-        code.putln("/* Runtime support code */")
-        code.putln("")
-        code.putln("static void %s(void) {" % Naming.fileinit_cname)
-        code.putln("%s = %s;" % 
-            (Naming.filetable_cname, Naming.filenames_cname))
-        code.putln("}")
-        code.globalstate.put_utility_code_defs(code)
-        code.put(PyrexTypes.type_conversion_functions)
-        code.putln("")
-
 #------------------------------------------------------------------------------------
 #
 #  Runtime support code