Introduced code.globalstate.use_utility_code and used it in Buffer.py

Cython/Compiler/Annotate.py

@@ -30,8 +30,8 @@ class AnnotationCCodeWriter(CCodeWriter):
             self.annotations = create_from.annotations
             self.code = create_from.code
 
-    def create_new(self, create_from, buffer):
-        return AnnotationCCodeWriter(create_from, buffer)
+    def create_new(self, create_from, buffer, copy_formatting):
+        return AnnotationCCodeWriter(create_from, buffer, copy_formatting)
 
     def write(self, s):
         CCodeWriter.write(self, s)

Cython/Compiler/Code.py

@@ -127,6 +127,7 @@ class FunctionState(object):
         freelist = self.temps_free.get(type)
         if freelist is None:
             freelist = []
+
             self.temps_free[type] = freelist
         freelist.append(name)
 
@@ -136,11 +137,15 @@ class GlobalState(object):
     # input_file_contents dict         contents (=list of lines) of any file that was used as input
     #                                  to create this output C code.  This is
     #                                  used to annotate the comments.
+    # used_utility_code set(string|int) Ids of used utility code (to avoid reinsertion)
+    # utilprotowriter CCodeWriter
+    # utildefwriter   CCodeWriter
 
     def __init__(self):
         self.filename_table = {}
         self.filename_list = []
         self.input_file_contents = {}
+        self.used_utility_code = set()
 
     def lookup_filename(self, filename):
         try:
@@ -162,6 +167,58 @@ class GlobalState(object):
             self.input_file_contents[source_desc] = F
             return F
 
+    def use_utility_code(self, codetup, name=None):
+        """
+        Adds the given utility code to the C file if needed.
+
+        codetup should unpack into one prototype code part and one
+        definition code part, both strings inserted directly in C.
+
+        If name is provided, it is used as an identifier to avoid inserting
+        code twice. Otherwise, id(codetup) is used as such an identifier.
+        """
+        if name is None: name = id(codetup)
+        if self.check_utility_code_needed_and_register(name):
+            proto, _def = codetup
+            self.utilprotowriter.put(proto)
+            self.utildefwriter.put(_def)
+
+    def has_utility_code(self, name):
+        return name in self.used_utility_code
+
+    def use_generated_code(self, func, name, *args, **kw):
+        """
+        Requests that the utility code that func can generate is used in the C
+        file. func is called like this:
+
+        func(proto, definition, name, *args, **kw)
+
+        where proto and definition are two CCodeWriter instances; the
+        former should have the prototype written to it and the other the definition.
+        
+        The call might happen at some later point (if compiling multiple modules
+        into a cache for instance), and will only happen once per utility code.
+
+        name is used to identify the utility code, so that it isn't regenerated
+        when the same code is requested again.
+        """
+        if self.check_utility_code_needed_and_register(name):
+            func(self.utilprotowriter, self.utildefwriter,
+                 name, *args, **kw)
+
+    def check_utility_code_needed_and_register(self, name):
+        if name in self.used_utility_code:
+            return False
+        else:
+            self.used_utility_code.add(name)
+            return True
+
+    def put_utility_code_protos(self, writer):
+        writer.insert(self.utilprotowriter)
+
+    def put_utility_code_defs(self, writer):
+        writer.insert(self.utildefwriter)
+
 
 def funccontext_property(name):
     def get(self):
@@ -198,29 +255,34 @@ class CCodeWriter(object):
     # globalstate      GlobalState     contains state global for a C file (input file info,
     #                                  utility code, declared constants etc.)
    
-    def __init__(self, create_from=None, buffer=None):
+    def __init__(self, create_from=None, buffer=None, copy_formatting=False):
         if buffer is None: buffer = StringIOTree()
         self.buffer = buffer
         self.marker = None
         self.last_marker_line = 0
         
-        self.funcstate = None # always start with no function state
-        if create_from is None:
-            # Root CCodeWriter
+        self.funcstate = None
         self.level = 0
         self.bol = 1
+        if create_from is None:
+            # Root CCodeWriter
             self.globalstate = GlobalState()
+            # These needs to be constructed after all state is set, as
+            # the construction copies over the state
+            self.globalstate.utilprotowriter = self.new_writer()
+            self.globalstate.utildefwriter = self.new_writer()
         else:
             # Use same global state
             self.globalstate = create_from.globalstate
             # Clone formatting state
+            if copy_formatting:
                 self.level = create_from.level
                 self.bol = create_from.bol
 
-    def create_new(self, create_from, buffer):
+    def create_new(self, create_from, buffer, copy_formatting):
         # polymorphic constructor -- very slightly more versatile
         # than using __class__
-        return CCodeWriter(create_from, buffer)
+        return CCodeWriter(create_from, buffer, copy_formatting)
 
     def copyto(self, f):
         self.buffer.copyto(f)
@@ -232,9 +294,25 @@ class CCodeWriter(object):
         self.buffer.write(s)
 
     def insertion_point(self):
-        other = self.create_new(create_from=self, buffer=self.buffer.insertion_point())
+        other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
         return other
 
+    def new_writer(self):
+        """
+        Creates a new CCodeWriter connected to the same global state, which
+        can later be inserted using insert.
+        """
+        return CCodeWriter(create_from=self)
+
+    def insert(self, writer):
+        """
+        Inserts the contents of another code writer (created with
+        the same global state) in the current location.
+
+        It is ok to write to the inserted writer also after insertion.
+        """
+        assert writer.globalstate is self.globalstate
+        self.buffer.insert(writer.buffer)
 
     # Properties delegated to function scope
     label_counter = funccontext_property("label_counter")

Cython/Compiler/ModuleNode.py

@@ -1950,6 +1950,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     type.typeptr_cname, type.typeobj_cname))
     
     def generate_utility_functions(self, env, code, h_code):
+        for codetup, name in env.utility_code_list:
+            code.globalstate.use_utility_code(codetup, name)
+       
+        code.globalstate.put_utility_code_protos(h_code)
         code.putln("")
         code.putln("/* Runtime support code */")
         code.putln("")
@@ -1957,9 +1961,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("%s = %s;" % 
             (Naming.filetable_cname, Naming.filenames_cname))
         code.putln("}")
-        for utility_code in env.utility_code_used:
-            h_code.put(utility_code[0])
-            code.put(utility_code[1])
+        code.globalstate.put_utility_code_defs(code)
         code.put(PyrexTypes.type_conversion_functions)
         code.putln("")
 

Cython/Compiler/Symtab.py

@@ -23,11 +23,12 @@ nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match
 class BufferAux:
     writable_needed = False
     
-    def __init__(self, buffer_info_var, stridevars, shapevars, tschecker):
+    def __init__(self, buffer_info_var, stridevars, shapevars,
+                 suboffsetvars):
         self.buffer_info_var = buffer_info_var
         self.stridevars = stridevars
         self.shapevars = shapevars
-        self.tschecker = tschecker
+        self.suboffsetvars = suboffsetvars
         
     def __repr__(self):
         return "<BufferAux %r>" % self.__dict__
@@ -621,9 +622,6 @@ class Scope:
     def use_utility_code(self, new_code, name=None):
         self.global_scope().use_utility_code(new_code, name)
 
-    def has_utility_code(self, name):
-        return self.global_scope().has_utility_code(name)
-
     def generate_library_function_declarations(self, code):
         # Generate extern decls for C library funcs used.
         #if self.pow_function_used:
@@ -742,8 +740,7 @@ class ModuleScope(Scope):
     # doc                  string             Module doc string
     # doc_cname            string             C name of module doc string
     # const_counter        integer            Counter for naming constants
-    # utility_code_used    [string]           Utility code to be included
-    # utility_code_names   set(string)        (Optional) names for named (often generated) utility code
+    # utility_code_list    [((string, string), string)] Queuing utility codes for forwarding to Code.py
     # default_entries      [Entry]            Function argument default entries
     # python_include_files [string]           Standard  Python headers to be included
     # include_files        [string]           Other C headers to be included
@@ -777,8 +774,7 @@ class ModuleScope(Scope):
         self.doc = ""
         self.doc_cname = Naming.moddoc_cname
         self.const_counter = 1
-        self.utility_code_used = []
-        self.utility_code_names = set()
+        self.utility_code_list = []
         self.default_entries = []
         self.module_entries = {}
         self.python_include_files = ["Python.h", "structmember.h"]
@@ -938,24 +934,7 @@ class ModuleScope(Scope):
         return "%s%s%d" % (Naming.const_prefix, prefix, n)
     
     def use_utility_code(self, new_code, name=None):
-        #  Add string to list of utility code to be included,
-        #  if not already there (tested using the provided name,
-        #  or 'is' if name=None -- if the utility code is dynamically
-        #  generated, use the name, otherwise it is not needed).
-        if name is not None:
-            if name in self.utility_code_names:
-                return
-        for old_code in self.utility_code_used:
-            if old_code is new_code:
-                return
-        self.utility_code_used.append(new_code)
-        self.utility_code_names.add(name)
-
-    def has_utility_code(self, name):
-        # Checks if utility code (that is registered by name) has
-        # previously been registered. This is useful if the utility code
-        # is dynamically generated to avoid re-generation.
-        return name in self.utility_code_names
+        self.utility_code_list.append((new_code, name))
 
     def declare_c_class(self, name, pos, defining = 0, implementing = 0,
         module_name = None, base_type = None, objstruct_cname = None,