Merge

Cython/Compiler/Annotate.py

@@ -30,8 +30,8 @@ class AnnotationCCodeWriter(CCodeWriter):
             self.annotations = create_from.annotations
             self.code = create_from.code
 
-    def create_new(self, create_from, buffer):
-        return AnnotationCCodeWriter(create_from, buffer)
+    def create_new(self, create_from, buffer, copy_formatting):
+        return AnnotationCCodeWriter(create_from, buffer, copy_formatting)
 
     def write(self, s):
         CCodeWriter.write(self, s)

Cython/Compiler/Buffer.py

@@ -33,7 +33,6 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
                 node.scope.include_files.append("endian.h")
             use_py2_buffer_functions(node.scope)
             use_empty_bufstruct_code(node.scope, self.max_ndim)
-            node.scope.use_utility_code(access_utility_code)
         return result
 
 
@@ -60,9 +59,6 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
             if buftype.ndim > self.max_ndim:
                 self.max_ndim = buftype.ndim
 
-            # Get or make a type string checker
-            tschecker = buffer_type_checker(buftype.dtype, scope)
-
             # Declare auxiliary vars
             cname = scope.mangle(Naming.bufstruct_prefix, name)
             bufinfo = scope.declare_var(name="$%s" % cname, cname=cname,
@@ -83,17 +79,13 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
 
             stridevars = [var(Naming.bufstride_prefix, i, "0") for i in range(entry.type.ndim)]
             shapevars = [var(Naming.bufshape_prefix, i, "0") for i in range(entry.type.ndim)]            
-            entry.buffer_aux = Symtab.BufferAux(bufinfo, stridevars, shapevars, tschecker)
             mode = entry.type.mode
             if mode == 'full':
                 suboffsetvars = [var(Naming.bufsuboffset_prefix, i, "-1") for i in range(entry.type.ndim)]
-                entry.buffer_aux.lookup = get_buf_lookup_full(scope, entry.type.ndim)
             elif mode == 'strided':
                 suboffsetvars = None
-                entry.buffer_aux.lookup = get_buf_lookup_strided(scope, entry.type.ndim)
 
-            entry.buffer_aux.suboffsetvars = suboffsetvars
-            entry.buffer_aux.get_buffer_cname = tschecker
+            entry.buffer_aux = Symtab.BufferAux(bufinfo, stridevars, shapevars, suboffsetvars)
             
         scope.buffer_entries = bufvars
         self.scope = scope
@@ -144,24 +136,19 @@ def put_unpack_buffer_aux_into_scope(buffer_aux, mode, code):
                              (s.cname, bufstruct, field, idx)
                              for idx, s in enumerate(vars)]))
 
-def getbuffer_cond_code(obj_cname, buffer_aux, flags, ndim):
-    bufstruct = buffer_aux.buffer_info_var.cname
-    return "%s(%s, &%s, %s, %d) == -1" % (
-        buffer_aux.get_buffer_cname, obj_cname, bufstruct, flags, ndim)
-                   
 def put_acquire_arg_buffer(entry, code, pos):
+    code.globalstate.use_utility_code(acquire_utility_code)
     buffer_aux = entry.buffer_aux
-    cname  = entry.cname
-    bufstruct = buffer_aux.buffer_info_var.cname
-    flags = get_flags(buffer_aux, entry.type)
+    getbuffer_cname = get_getbuffer_code(entry.type.dtype, code)
     # Acquire any new buffer
-    code.putln(code.error_goto_if(getbuffer_cond_code(cname,
-                                                    buffer_aux,
-                                                    flags,
-                                                    entry.type.ndim),
-                                pos))
+    code.putln(code.error_goto_if("%s(%s, &%s, %s, %d) == -1" % (
+        getbuffer_cname,
+        entry.cname,
+        entry.buffer_aux.buffer_info_var.cname,
+        get_flags(buffer_aux, entry.type),
+        entry.type.ndim), pos))
     # An exception raised in arg parsing cannot be catched, so no
-    # need to do care about the buffer then.
+    # need to care about the buffer then.
     put_unpack_buffer_aux_into_scope(buffer_aux, entry.type.mode, code)
 
 #def put_release_buffer_normal(entry, code):
@@ -192,11 +179,12 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buffer_aux, buffer_type,
       (which may or may not succeed).
     """
     
+    code.globalstate.use_utility_code(acquire_utility_code)
     bufstruct = buffer_aux.buffer_info_var.cname
     flags = get_flags(buffer_aux, buffer_type)
 
-    getbuffer = "%s(%%s, &%s, %s, %d)" % (buffer_aux.get_buffer_cname,
-                                          # note: object is filled in later
+    getbuffer = "%s(%%s, &%s, %s, %d)" % (get_getbuffer_code(buffer_type.dtype, code),
+                                          # note: object is filled in later (%%s)
                                           bufstruct,
                                           flags,
                                           buffer_type.ndim)
@@ -209,7 +197,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buffer_aux, buffer_type,
             lhs_cname, bufstruct))
         code.end_block()
         # Acquire
-        retcode_cname = code.func.allocate_temp(PyrexTypes.c_int_type)
+        retcode_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type)
         code.putln("%s = %s;" % (retcode_cname, getbuffer % rhs_cname))
         code.putln('if (%s) ' % (code.unlikely("%s < 0" % retcode_cname)))
         # If acquisition failed, attempt to reacquire the old buffer
@@ -217,7 +205,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buffer_aux, buffer_type,
         # will cause the reacquisition exception to be reported, one
         # can consider working around this later.
         code.begin_block()
-        type, value, tb = [code.func.allocate_temp(PyrexTypes.py_object_type)
+        type, value, tb = [code.funcstate.allocate_temp(PyrexTypes.py_object_type)
                            for i in range(3)]
         code.putln('PyErr_Fetch(&%s, &%s, &%s);' % (type, value, tb))
         code.put('if (%s) ' % code.unlikely("%s == -1" % (getbuffer % lhs_cname)))
@@ -227,13 +215,13 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buffer_aux, buffer_type,
         code.putln('} else {')
         code.putln('PyErr_Restore(%s, %s, %s);' % (type, value, tb))
         for t in (type, value, tb):
-            code.func.release_temp(t)
+            code.funcstate.release_temp(t)
         code.end_block()
         # Unpack indices
         code.end_block()
         put_unpack_buffer_aux_into_scope(buffer_aux, buffer_type.mode, code)
         code.putln(code.error_goto_if_neg(retcode_cname, pos))
-        code.func.release_temp(retcode_cname)
+        code.funcstate.release_temp(retcode_cname)
     else:
         # Our entry had no previous value, so set to None when acquisition fails.
         # In this case, auxiliary vars should be set up right in initialization to a zero-buffer,
@@ -256,12 +244,14 @@ def put_access(entry, index_signeds, index_cnames, pos, code):
     body. The lookup however is delegated to a inline function that is instantiated
     once per ndim (lookup with suboffsets tend to get quite complicated).
     """
+    code.globalstate.use_utility_code(access_utility_code)
     bufaux = entry.buffer_aux
     bufstruct = bufaux.buffer_info_var.cname
     # Check bounds and fix negative indices
     boundscheck = True
     nonegs = True
-    tmp_cname = code.func.allocate_temp(PyrexTypes.c_int_type)
+    tmp_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type)
+
     if boundscheck:
         code.putln("%s = -1;" % tmp_cname)
     for idx, (signed, cname, shape) in enumerate(zip(index_signeds, index_cnames,
@@ -288,21 +278,33 @@ def put_access(entry, index_signeds, index_cnames, pos, code):
         code.putln('__Pyx_BufferIndexError(%s);' % tmp_cname)
         code.putln(code.error_goto(pos))
         code.end_block()
-    code.func.release_temp(tmp_cname)
+    code.funcstate.release_temp(tmp_cname)
+
+
+
+    
 
     # Create buffer lookup and return it
     params = []
+    nd = entry.type.ndim
     if entry.type.mode == 'full':
         for i, s, o in zip(index_cnames, bufaux.stridevars, bufaux.suboffsetvars):
             params.append(i)
             params.append(s.cname)
             params.append(o.cname)
+
+        funcname = "__Pyx_BufPtrFull%dd" % nd
+        funcgen = buf_lookup_full_code
     else:
         for i, s in zip(index_cnames, bufaux.stridevars):
             params.append(i)
             params.append(s.cname)
-    ptrcode = "%s(%s.buf, %s)" % (bufaux.lookup, bufstruct, 
-                          ", ".join(params))
+        funcname = "__Pyx_BufPtrStrided%dd" % nd
+        funcgen = buf_lookup_strided_code
+        
+    code.globalstate.use_generated_code(funcgen, name=funcname, nd=nd)
+
+    ptrcode = "%s(%s.buf, %s)" % (funcname, bufstruct, ", ".join(params))
     valuecode = "*%s" % entry.type.buffer_ptr_type.cast_code(ptrcode)
     return valuecode
 
@@ -313,54 +315,35 @@ def use_empty_bufstruct_code(env, max_ndim):
         Py_ssize_t __Pyx_zeros[] = {%s};
         Py_ssize_t __Pyx_minusones[] = {%s};
     """) % (", ".join(["0"] * max_ndim), ", ".join(["-1"] * max_ndim))
-    env.use_utility_code([code, ""])
+    env.use_utility_code([code, ""], "empty_bufstruct_code")
 
 
-def get_buf_lookup_strided(env, nd):
+def buf_lookup_strided_code(proto, defin, name, nd):
     """
-    Generates and registers as utility a buffer lookup function for the right number
+    Generates a buffer lookup function for the right number
     of dimensions. The function gives back a void* at the right location.
     """
-    name = "__Pyx_BufPtrStrided_%dd" % nd
-    if not env.has_utility_code(name):
-        # _i_ndex, _s_tride
-        args = ", ".join(["i%d, s%d" % (i, i) for i in range(nd)])
-        offset = " + ".join(["i%d * s%d" % (i, i) for i in range(nd)])
-        proto = dedent("""\
-        #define %s(buf, %s) ((char*)buf + %s)
-        """) % (name, args, offset) 
-        env.use_utility_code([proto, ""], name=name)
-        
-    return name
+    # _i_ndex, _s_tride
+    args = ", ".join(["i%d, s%d" % (i, i) for i in range(nd)])
+    offset = " + ".join(["i%d * s%d" % (i, i) for i in range(nd)])
+    proto.putln("#define %s(buf, %s) ((char*)buf + %s)" % (name, args, offset))
 
-
-def get_buf_lookup_full(env, nd):
+def buf_lookup_full_code(proto, defin, name, nd):
     """
-    Generates and registers as utility a buffer lookup function for the right number
+    Generates a buffer lookup function for the right number
     of dimensions. The function gives back a void* at the right location.
     """
-    name = "__Pyx_BufPtrFull_%dd" % nd
-    if not env.has_utility_code(name):
-        # _i_ndex, _s_tride, sub_o_ffset
-        args = ", ".join(["Py_ssize_t i%d, Py_ssize_t s%d, Py_ssize_t o%d" % (i, i, i) for i in range(nd)])
-        proto = dedent("""\
-        static INLINE void* %s(void* buf, %s);
-        """) % (name, args) 
-
-        func = dedent("""
+    # _i_ndex, _s_tride, sub_o_ffset
+    args = ", ".join(["Py_ssize_t i%d, Py_ssize_t s%d, Py_ssize_t o%d" % (i, i, i) for i in range(nd)])
+    proto.putln("static INLINE void* %s(void* buf, %s);" % (name, args))
+    defin.putln(dedent("""
         static INLINE void* %s(void* buf, %s) {
           char* ptr = (char*)buf;
         """) % (name, args) + "".join([dedent("""\
           ptr += s%d * i%d;
           if (o%d >= 0) ptr = *((char**)ptr) + o%d; 
         """) % (i, i, i, i) for i in range(nd)]
-        ) + "\nreturn ptr;\n}"
-
-        env.use_utility_code([proto, func], name=name)
-        
-    return name
-
-
+        ) + "\nreturn ptr;\n}")
 
 
 #
@@ -375,11 +358,11 @@ def mangle_dtype_name(dtype):
         prefix = ""
     return prefix + dtype.declaration_code("").replace(" ", "_")
 
-def get_ts_check_item(dtype, env):
+def get_ts_check_item(dtype, writer):
     # See if we can consume one (unnamed) dtype as next item
     # Put native types and structs in seperate namespaces (as one could create a struct named unsigned_int...)
     name = "__Pyx_BufferTypestringCheck_item_%s" % mangle_dtype_name(dtype)
-    if not env.has_utility_code(name):
+    if not writer.globalstate.has_utility_code(name):
         char = dtype.typestring
         if char is not None:
                 # Can use direct comparison
@@ -415,7 +398,7 @@ def get_ts_check_item(dtype, env):
                       return NULL;
                     } else return ts + 1;
                 """, 2)
-        env.use_utility_code([dedent("""\
+        writer.globalstate.use_utility_code([dedent("""\
             static const char* %s(const char* ts); /*proto*/
         """) % name, dedent("""
             static const char* %s(const char* ts) {
@@ -425,7 +408,7 @@ def get_ts_check_item(dtype, env):
 
     return name
 
-def get_getbuffer_code(dtype, env):
+def get_getbuffer_code(dtype, code):
     """
     Generate a utility function for getting a buffer for the given dtype.
     The function will:
@@ -436,9 +419,9 @@ def get_getbuffer_code(dtype, env):
     """
 
     name = "__Pyx_GetBuffer_%s" % mangle_dtype_name(dtype)
-    if not env.has_utility_code(name):
-        env.use_utility_code(acquire_utility_code)
-        itemchecker = get_ts_check_item(dtype, env)
+    if not code.globalstate.has_utility_code(name):
+        code.globalstate.use_utility_code(acquire_utility_code)
+        itemchecker = get_ts_check_item(dtype, code)
         utilcode = [dedent("""
         static int %s(PyObject* obj, Py_buffer* buf, int flags, int nd); /*proto*/
         """) % name, dedent("""
@@ -473,72 +456,21 @@ def get_getbuffer_code(dtype, env):
           __Pyx_ZeroBuffer(buf);
           return -1;
         }""") % locals()]
-        env.use_utility_code(utilcode, name)
+        code.globalstate.use_utility_code(utilcode, name)
     return name
 
-def buffer_type_checker(dtype, env):
+def buffer_type_checker(dtype, code):
     # Creates a type checker function for the given type.
     if dtype.is_struct_or_union:
         assert False
     elif dtype.is_int or dtype.is_float:
         # This includes simple typedef-ed types
-        funcname = get_getbuffer_code(dtype, env)
+        funcname = get_getbuffer_code(dtype, code)
     else:
         assert False
     return funcname
 
 def use_py2_buffer_functions(env):
-    # will be refactored
-    try:
-        env.entries[u'numpy']
-        env.use_utility_code(["","""
-static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags) {
-  /* This function is always called after a type-check; safe to cast */
-  PyArrayObject *arr = (PyArrayObject*)obj;
-  PyArray_Descr *type = (PyArray_Descr*)arr->descr;
-
-  
-  int typenum = PyArray_TYPE(obj);
-  if (!PyTypeNum_ISNUMBER(typenum)) {
-    PyErr_Format(PyExc_TypeError, "Only numeric NumPy types currently supported.");
-    return -1;
-  }
-
-  /*
-  NumPy format codes doesn't completely match buffer codes;
-  seems safest to retranslate.
-                            01234567890123456789012345*/
-  const char* base_codes = "?bBhHiIlLqQfdgfdgO";
-
-  char* format = (char*)malloc(4);
-  char* fp = format;
-  *fp++ = type->byteorder;
-  if (PyTypeNum_ISCOMPLEX(typenum)) *fp++ = 'Z';
-  *fp++ = base_codes[typenum];
-  *fp = 0;
-
-  view->buf = arr->data;
-  view->readonly = !PyArray_ISWRITEABLE(obj);
-  view->ndim = PyArray_NDIM(arr);
-  view->strides = PyArray_STRIDES(arr);
-  view->shape = PyArray_DIMS(arr);
-  view->suboffsets = NULL;
-  view->format = format;
-  view->itemsize = type->elsize;
-
-  view->internal = 0;
-  return 0;
-}
-
-static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
-  free((char*)view->format);
-  view->format = NULL;
-}
-
-"""])
-    except KeyError:
-        pass
-
     codename = "PyObject_GetBuffer" # just a representative unique key
 
     # Search all types for __getbuffer__ overloads
@@ -562,6 +494,7 @@ static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
     try:
         ndarrtype = env.entries[u'numpy'].as_module.entries['ndarray'].type
         types.append((ndarrtype.typeptr_cname, "numpy_getbuffer", "numpy_releasebuffer"))
+        env.use_utility_code(numpy_code)
     except KeyError:
         pass
 
@@ -602,7 +535,7 @@ static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
         static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags);
         static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view);
         #endif
-    """) ,code], codename)
+    """), code], codename)
 
 #
 # Static utility code
@@ -690,3 +623,53 @@ static void __Pyx_RaiseBufferFallbackError(void) {
 }
 
 """]
+
+
+numpy_code = ["""
+static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags);
+static void numpy_releasebuffer(PyObject *obj, Py_buffer *view);
+""","""
+static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags) {
+  /* This function is always called after a type-check; safe to cast */
+  PyArrayObject *arr = (PyArrayObject*)obj;
+  PyArray_Descr *type = (PyArray_Descr*)arr->descr;
+
+  
+  int typenum = PyArray_TYPE(obj);
+  if (!PyTypeNum_ISNUMBER(typenum)) {
+    PyErr_Format(PyExc_TypeError, "Only numeric NumPy types currently supported.");
+    return -1;
+  }
+
+  /*
+  NumPy format codes doesn't completely match buffer codes;
+  seems safest to retranslate.
+                            01234567890123456789012345*/
+  const char* base_codes = "?bBhHiIlLqQfdgfdgO";
+
+  char* format = (char*)malloc(4);
+  char* fp = format;
+  *fp++ = type->byteorder;
+  if (PyTypeNum_ISCOMPLEX(typenum)) *fp++ = 'Z';
+  *fp++ = base_codes[typenum];
+  *fp = 0;
+
+  view->buf = arr->data;
+  view->readonly = !PyArray_ISWRITEABLE(obj);
+  view->ndim = PyArray_NDIM(arr);
+  view->strides = PyArray_STRIDES(arr);
+  view->shape = PyArray_DIMS(arr);
+  view->suboffsets = NULL;
+  view->format = format;
+  view->itemsize = type->elsize;
+
+  view->internal = 0;
+  return 0;
+}
+
+static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
+  free((char*)view->format);
+  view->format = NULL;
+}
+
+"""]

Cython/Compiler/Code.py

@@ -12,7 +12,16 @@ from Scanning import SourceDescriptor
 from Cython.StringIOTree import StringIOTree
 from sets import Set as set
 
-class FunctionContext(object):
+class FunctionState(object):
+    # return_label     string          function return point label
+    # error_label      string          error catch point label
+    # continue_label   string          loop continue point label
+    # break_label      string          loop break point label
+    # return_from_error_cleanup_label string
+    # label_counter    integer         counter for naming labels
+    # in_try_finally   boolean         inside try of try...finally
+    # exc_vars         (string * 3)    exception variables for reraise, or None
+
     # Not used for now, perhaps later
     def __init__(self, names_taken=set()):
         self.names_taken = names_taken
@@ -25,6 +34,9 @@ class FunctionContext(object):
         self.continue_label = None
         self.break_label = None
 
+        self.in_try_finally = 0
+        self.exc_vars = None
+
         self.temps_allocated = [] # of (name, type)
         self.temps_free = {} # type -> list of free vars
         self.temps_used_type = {} # name -> type
@@ -115,14 +127,104 @@ class FunctionContext(object):
         freelist = self.temps_free.get(type)
         if freelist is None:
             freelist = []
+
             self.temps_free[type] = freelist
         freelist.append(name)
 
+class GlobalState(object):
+    # filename_table   {string : int}  for finding filename table indexes
+    # filename_list    [string]        filenames in filename table order
+    # input_file_contents dict         contents (=list of lines) of any file that was used as input
+    #                                  to create this output C code.  This is
+    #                                  used to annotate the comments.
+    # used_utility_code set(string|int) Ids of used utility code (to avoid reinsertion)
+    # utilprotowriter CCodeWriter
+    # utildefwriter   CCodeWriter
+
+    def __init__(self):
+        self.filename_table = {}
+        self.filename_list = []
+        self.input_file_contents = {}
+        self.used_utility_code = set()
+
+    def lookup_filename(self, filename):
+        try:
+            index = self.filename_table[filename]
+        except KeyError:
+            index = len(self.filename_list)
+            self.filename_list.append(filename)
+            self.filename_table[filename] = index
+        return index
+
+    def commented_file_contents(self, source_desc):
+        try:
+            return self.input_file_contents[source_desc]
+        except KeyError:
+            F = [u' * ' + line.rstrip().replace(
+                    u'*/', u'*[inserted by cython to avoid comment closer]/'
+                    ).encode('ASCII', 'replace') # + Py2 auto-decode to unicode
+                 for line in source_desc.get_lines()]
+            self.input_file_contents[source_desc] = F
+            return F
+
+    def use_utility_code(self, codetup, name=None):
+        """
+        Adds the given utility code to the C file if needed.
+
+        codetup should unpack into one prototype code part and one
+        definition code part, both strings inserted directly in C.
+
+        If name is provided, it is used as an identifier to avoid inserting
+        code twice. Otherwise, id(codetup) is used as such an identifier.
+        """
+        if name is None: name = id(codetup)
+        if self.check_utility_code_needed_and_register(name):
+            proto, _def = codetup
+            self.utilprotowriter.put(proto)
+            self.utildefwriter.put(_def)
+
+    def has_utility_code(self, name):
+        return name in self.used_utility_code
+
+    def use_generated_code(self, func, name, *args, **kw):
+        """
+        Requests that the utility code that func can generate is used in the C
+        file. func is called like this:
+
+        func(proto, definition, name, *args, **kw)
+
+        where proto and definition are two CCodeWriter instances; the
+        former should have the prototype written to it and the other the definition.
+        
+        The call might happen at some later point (if compiling multiple modules
+        into a cache for instance), and will only happen once per utility code.
+
+        name is used to identify the utility code, so that it isn't regenerated
+        when the same code is requested again.
+        """
+        if self.check_utility_code_needed_and_register(name):
+            func(self.utilprotowriter, self.utildefwriter,
+                 name, *args, **kw)
+
+    def check_utility_code_needed_and_register(self, name):
+        if name in self.used_utility_code:
+            return False
+        else:
+            self.used_utility_code.add(name)
+            return True
+
+    def put_utility_code_protos(self, writer):
+        writer.insert(self.utilprotowriter)
+
+    def put_utility_code_defs(self, writer):
+        writer.insert(self.utildefwriter)
+
+
 def funccontext_property(name):
     def get(self):
-        return getattr(self.func, name)
+        return getattr(self.funcstate, name)
     def set(self, value):
-        setattr(self.func, name, value)
+        setattr(self.funcstate, name, value)
     return property(get, set)
 
 class CCodeWriter(object):
@@ -148,52 +250,39 @@ class CCodeWriter(object):
     # level            int             indentation level
     # bol              bool            beginning of line?
     # marker           string          comment to emit before next line
-    # return_label     string          function return point label
-    # error_label      string          error catch point label
-    # continue_label   string          loop continue point label
-    # break_label      string          loop break point label
-    # return_from_error_cleanup_label string
-    # label_counter    integer         counter for naming labels
-    # in_try_finally   boolean         inside try of try...finally
-    # filename_table   {string : int}  for finding filename table indexes
-    # filename_list    [string]        filenames in filename table order
-    # exc_vars         (string * 3)    exception variables for reraise, or None
-    # input_file_contents dict         contents (=list of lines) of any file that was used as input
-    #                                  to create this output C code.  This is
-    #                                  used to annotate the comments.
-    # func             FunctionContext contains labels and temps context info
+    # funcstate        FunctionState   contains state local to a C function used for code
+    #                                  generation (labels and temps state etc.)
+    # globalstate      GlobalState     contains state global for a C file (input file info,
+    #                                  utility code, declared constants etc.)
    
-    in_try_finally = 0
-    
-    def __init__(self, create_from=None, buffer=None):
+    def __init__(self, create_from=None, buffer=None, copy_formatting=False):
         if buffer is None: buffer = StringIOTree()
         self.buffer = buffer
         self.marker = None
         self.last_marker_line = 0
-        self.func = None
+        
+        self.funcstate = None
+        self.level = 0
+        self.bol = 1
         if create_from is None:
             # Root CCodeWriter
-            self.level = 0
-            self.bol = 1
-            self.filename_table = {}
-            self.filename_list = []
-            self.exc_vars = None
-            self.input_file_contents = {}
+            self.globalstate = GlobalState()
+            # These needs to be constructed after all state is set, as
+            # the construction copies over the state
+            self.globalstate.utilprotowriter = self.new_writer()
+            self.globalstate.utildefwriter = self.new_writer()
         else:
+            # Use same global state
+            self.globalstate = create_from.globalstate
             # Clone formatting state
-            c = create_from
-            self.level = c.level
-            self.bol = c.bol
-            # Note: NOT copying but sharing instance
-            self.filename_table = c.filename_table
-            self.filename_list = []
-            self.input_file_contents = c.input_file_contents
-            # Leave other state alone
-
-    def create_new(self, create_from, buffer):
+            if copy_formatting:
+                self.level = create_from.level
+                self.bol = create_from.bol
+
+    def create_new(self, create_from, buffer, copy_formatting):
         # polymorphic constructor -- very slightly more versatile
         # than using __class__
-        return CCodeWriter(create_from, buffer)
+        return CCodeWriter(create_from, buffer, copy_formatting)
 
     def copyto(self, f):
         self.buffer.copyto(f)
@@ -205,9 +294,25 @@ class CCodeWriter(object):
         self.buffer.write(s)
 
     def insertion_point(self):
-        other = self.create_new(create_from=self, buffer=self.buffer.insertion_point())
+        other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
         return other
 
+    def new_writer(self):
+        """
+        Creates a new CCodeWriter connected to the same global state, which
+        can later be inserted using insert.
+        """
+        return CCodeWriter(create_from=self)
+
+    def insert(self, writer):
+        """
+        Inserts the contents of another code writer (created with
+        the same global state) in the current location.
+
+        It is ok to write to the inserted writer also after insertion.
+        """
+        assert writer.globalstate is self.globalstate
+        self.buffer.insert(writer.buffer)
 
     # Properties delegated to function scope
     label_counter = funccontext_property("label_counter")
@@ -216,26 +321,26 @@ class CCodeWriter(object):
     labels_used = funccontext_property("labels_used")
     continue_label = funccontext_property("continue_label")
     break_label = funccontext_property("break_label")
-
+    return_from_error_cleanup_label = funccontext_property("return_from_error_cleanup_label")
 
     # Functions delegated to function scope
-    def new_label(self):               return self.func.new_label()
-    def new_error_label(self):         return self.func.new_error_label()
-    def get_loop_labels(self):         return self.func.get_loop_labels()
-    def set_loop_labels(self, labels): return self.func.set_loop_labels(labels)
-    def new_loop_labels(self):         return self.func.new_loop_labels()
-    def get_all_labels(self):          return self.func.get_all_labels()
-    def set_all_labels(self, labels):  return self.func.set_all_labels(labels)
-    def all_new_labels(self):          return self.func.all_new_labels()
-    def use_label(self, lbl):          return self.func.use_label(lbl)
-    def label_used(self, lbl):         return self.func.label_used(lbl)
+    def new_label(self):               return self.funcstate.new_label()
+    def new_error_label(self):         return self.funcstate.new_error_label()
+    def get_loop_labels(self):         return self.funcstate.get_loop_labels()
+    def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels)
+    def new_loop_labels(self):         return self.funcstate.new_loop_labels()
+    def get_all_labels(self):          return self.funcstate.get_all_labels()
+    def set_all_labels(self, labels):  return self.funcstate.set_all_labels(labels)
+    def all_new_labels(self):          return self.funcstate.all_new_labels()
+    def use_label(self, lbl):          return self.funcstate.use_label(lbl)
+    def label_used(self, lbl):         return self.funcstate.label_used(lbl)
 
 
     def enter_cfunc_scope(self):
-        self.func = FunctionContext()
+        self.funcstate = FunctionState()
     
     def exit_cfunc_scope(self):
-        self.func = None
+        self.funcstate = None
 
     def putln(self, code = ""):
         if self.marker and self.bol:
@@ -287,17 +392,6 @@ class CCodeWriter(object):
     def get_py_version_hex(self, pyversion):
         return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]
 
-    def commented_file_contents(self, source_desc):
-        try:
-            return self.input_file_contents[source_desc]
-        except KeyError:
-            F = [u' * ' + line.rstrip().replace(
-                    u'*/', u'*[inserted by cython to avoid comment closer]/'
-                    ).encode('ASCII', 'replace') # + Py2 auto-decode to unicode
-                 for line in source_desc.get_lines()]
-            self.input_file_contents[source_desc] = F
-            return F
-
     def mark_pos(self, pos):
         if pos is None:
             return
@@ -305,7 +399,7 @@ class CCodeWriter(object):
         if self.last_marker_line == line:
             return
         assert isinstance(source_desc, SourceDescriptor)
-        contents = self.commented_file_contents(source_desc)
+        contents = self.globalstate.commented_file_contents(source_desc)
 
         lines = contents[max(0,line-3):line] # line numbers start at 1
         lines[-1] += u'             # <<<<<<<<<<<<<<'
@@ -317,11 +411,11 @@ class CCodeWriter(object):
 
         
     def put_label(self, lbl):
-        if lbl in self.func.labels_used:
+        if lbl in self.funcstate.labels_used:
             self.putln("%s:;" % lbl)
     
     def put_goto(self, lbl):
-        self.func.use_label(lbl)
+        self.funcstate.use_label(lbl)
         self.putln("goto %s;" % lbl)
     
     def put_var_declarations(self, entries, static = 0, dll_linkage = None,
@@ -481,8 +575,8 @@ class CCodeWriter(object):
             return cond
         
     def error_goto(self, pos):
-        lbl = self.func.error_label
-        self.func.use_label(lbl)
+        lbl = self.funcstate.error_label
+        self.funcstate.use_label(lbl)
         if Options.c_line_in_traceback:
             cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
         else:
@@ -509,13 +603,7 @@ class CCodeWriter(object):
         return self.error_goto_if("PyErr_Occurred()", pos)
     
     def lookup_filename(self, filename):
-        try:
-            index = self.filename_table[filename]
-        except KeyError:
-            index = len(self.filename_list)
-            self.filename_list.append(filename)
-            self.filename_table[filename] = index
-        return index
+        return self.globalstate.lookup_filename(filename)
 
 
 class PyrexCodeWriter:

Cython/Compiler/ExprNodes.py

@@ -1065,7 +1065,7 @@ class NameNode(AtomicExprNode):
             rhs.generate_post_assignment_code(code)
 
     def generate_acquire_buffer(self, rhs, code):
-        rhstmp = code.func.allocate_temp(self.entry.type)
+        rhstmp = code.funcstate.allocate_temp(self.entry.type)
         buffer_aux = self.entry.buffer_aux
         bufstruct = buffer_aux.buffer_info_var.cname
         code.putln('%s = %s;' % (rhstmp, rhs.result_as(self.ctype())))
@@ -1075,7 +1075,7 @@ class NameNode(AtomicExprNode):
                                     is_initialized=not self.skip_assignment_decref,
                                     pos=self.pos, code=code)
         code.putln("%s = 0;" % rhstmp)
-        code.func.release_temp(rhstmp)
+        code.funcstate.release_temp(rhstmp)
     
     def generate_deletion_code(self, code):
         if self.entry is None:
@@ -1520,7 +1520,7 @@ class IndexNode(ExprNode):
 
     def buffer_access_code(self, code):
         # Assign indices to temps
-        index_temps = [code.func.allocate_temp(i.type) for i in self.indices]
+        index_temps = [code.funcstate.allocate_temp(i.type) for i in self.indices]
         for temp, index in zip(index_temps, self.indices):
             code.putln("%s = %s;" % (temp, index.result_code))
         # Generate buffer access code using these temps

Cython/Compiler/ModuleNode.py

@@ -529,8 +529,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     def generate_filename_table(self, code):
         code.putln("")
         code.putln("static const char *%s[] = {" % Naming.filenames_cname)
-        if code.filename_list:
-            for source_desc in code.filename_list:
+        if code.globalstate.filename_list:
+            for source_desc in code.globalstate.filename_list:
                 filename = os.path.basename(source_desc.get_filenametable_entry())
                 escaped_filename = filename.replace("\\", "\\\\").replace('"', r'\"')
                 code.putln('"%s",' % 
@@ -1610,7 +1610,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln('}')
         
         tempdecl_code.put_var_declarations(env.temp_entries)
-        tempdecl_code.put_temp_declarations(code.func)
+        tempdecl_code.put_temp_declarations(code.funcstate)
 
         code.exit_cfunc_scope()
 
@@ -1950,6 +1950,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     type.typeptr_cname, type.typeobj_cname))
     
     def generate_utility_functions(self, env, code, h_code):
+        for codetup, name in env.utility_code_list:
+            code.globalstate.use_utility_code(codetup, name)
+       
+        code.globalstate.put_utility_code_protos(h_code)
         code.putln("")
         code.putln("/* Runtime support code */")
         code.putln("")
@@ -1957,9 +1961,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("%s = %s;" % 
             (Naming.filetable_cname, Naming.filenames_cname))
         code.putln("}")
-        for utility_code in env.utility_code_used:
-            h_code.put(utility_code[0])
-            code.put(utility_code[1])
+        code.globalstate.put_utility_code_defs(code)
         code.put(PyrexTypes.type_conversion_functions)
         code.putln("")
 

Cython/Compiler/Nodes.py

@@ -969,7 +969,7 @@ class FuncDefNode(StatNode, BlockNode):
         code.putln("}")
         # ----- Go back and insert temp variable declarations
         tempvardecl_code.put_var_declarations(lenv.temp_entries)
-        tempvardecl_code.put_temp_declarations(code.func)
+        tempvardecl_code.put_temp_declarations(code.funcstate)
         # ----- Python version
         code.exit_cfunc_scope()
         if self.py_func:
@@ -2634,7 +2634,7 @@ class ContinueStatNode(StatNode):
         pass
     
     def generate_execution_code(self, code):
-        if code.in_try_finally:
+        if code.funcstate.in_try_finally:
             error(self.pos, "continue statement inside try of try...finally")
         elif not code.continue_label:
             error(self.pos, "continue statement not inside loop")
@@ -2797,7 +2797,7 @@ class ReraiseStatNode(StatNode):
     gil_message = "Raising exception"
 
     def generate_execution_code(self, code):
-        vars = code.exc_vars
+        vars = code.funcstate.exc_vars
         if vars:
             code.putln("__Pyx_Raise(%s, %s, %s);" % tuple(vars))
             code.putln(code.error_goto(self.pos))
@@ -3478,10 +3478,10 @@ class ExceptClauseNode(Node):
             self.excinfo_tuple.generate_evaluation_code(code)
             self.excinfo_target.generate_assignment_code(self.excinfo_tuple, code)
 
-        old_exc_vars = code.exc_vars
-        code.exc_vars = self.exc_vars
+        old_exc_vars = code.funcstate.exc_vars
+        code.funcstate.exc_vars = self.exc_vars
         self.body.generate_execution_code(code)
-        code.exc_vars = old_exc_vars
+        code.funcstate.exc_vars = old_exc_vars
         for var in self.exc_vars:
             code.putln("Py_DECREF(%s); %s = 0;" % (var, var))
         code.put_goto(end_label)
@@ -3556,11 +3556,11 @@ class TryFinallyStatNode(StatNode):
         code.putln(
             "/*try:*/ {")
         if self.disallow_continue_in_try_finally:
-            was_in_try_finally = code.in_try_finally
-            code.in_try_finally = 1
+            was_in_try_finally = code.funcstate.in_try_finally
+            code.funcstate.in_try_finally = 1
         self.body.generate_execution_code(code)
         if self.disallow_continue_in_try_finally:
-            code.in_try_finally = was_in_try_finally
+            code.funcstate.in_try_finally = was_in_try_finally
         code.putln(
             "}")
         code.putln(

Cython/Compiler/Symtab.py

@@ -23,11 +23,12 @@ nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match
 class BufferAux:
     writable_needed = False
     
-    def __init__(self, buffer_info_var, stridevars, shapevars, tschecker):
+    def __init__(self, buffer_info_var, stridevars, shapevars,
+                 suboffsetvars):
         self.buffer_info_var = buffer_info_var
         self.stridevars = stridevars
         self.shapevars = shapevars
-        self.tschecker = tschecker
+        self.suboffsetvars = suboffsetvars
         
     def __repr__(self):
         return "<BufferAux %r>" % self.__dict__
@@ -620,9 +621,6 @@ class Scope:
     
     def use_utility_code(self, new_code, name=None):
         self.global_scope().use_utility_code(new_code, name)
-    
-    def has_utility_code(self, name):
-        return self.global_scope().has_utility_code(name)
 
     def generate_library_function_declarations(self, code):
         # Generate extern decls for C library funcs used.
@@ -742,8 +740,7 @@ class ModuleScope(Scope):
     # doc                  string             Module doc string
     # doc_cname            string             C name of module doc string
     # const_counter        integer            Counter for naming constants
-    # utility_code_used    [string]           Utility code to be included
-    # utility_code_names   set(string)        (Optional) names for named (often generated) utility code
+    # utility_code_list    [((string, string), string)] Queuing utility codes for forwarding to Code.py
     # default_entries      [Entry]            Function argument default entries
     # python_include_files [string]           Standard  Python headers to be included
     # include_files        [string]           Other C headers to be included
@@ -777,8 +774,7 @@ class ModuleScope(Scope):
         self.doc = ""
         self.doc_cname = Naming.moddoc_cname
         self.const_counter = 1
-        self.utility_code_used = []
-        self.utility_code_names = set()
+        self.utility_code_list = []
         self.default_entries = []
         self.module_entries = {}
         self.python_include_files = ["Python.h", "structmember.h"]
@@ -938,25 +934,8 @@ class ModuleScope(Scope):
         return "%s%s%d" % (Naming.const_prefix, prefix, n)
     
     def use_utility_code(self, new_code, name=None):
-        #  Add string to list of utility code to be included,
-        #  if not already there (tested using the provided name,
-        #  or 'is' if name=None -- if the utility code is dynamically
-        #  generated, use the name, otherwise it is not needed).
-        if name is not None:
-            if name in self.utility_code_names:
-                return
-        for old_code in self.utility_code_used:
-            if old_code is new_code:
-                return
-        self.utility_code_used.append(new_code)
-        self.utility_code_names.add(name)
-
-    def has_utility_code(self, name):
-        # Checks if utility code (that is registered by name) has
-        # previously been registered. This is useful if the utility code
-        # is dynamically generated to avoid re-generation.
-        return name in self.utility_code_names
-    
+        self.utility_code_list.append((new_code, name))
+
     def declare_c_class(self, name, pos, defining = 0, implementing = 0,
         module_name = None, base_type = None, objstruct_cname = None,
         typeobj_cname = None, visibility = 'private', typedef_flag = 0, api = 0):

Cython/StringIOTree.py

@@ -7,8 +7,7 @@ class StringIOTree(object):
 
     def __init__(self, stream=None):
         self.prepended_children = []
-        if stream is None: stream = StringIO()
-        self.stream = stream
+        self.stream = stream # if set to None, it will be constructed on first write
 
     def getvalue(self):
         return ("".join([x.getvalue() for x in self.prepended_children]) +
@@ -19,20 +18,44 @@ class StringIOTree(object):
         needs to happen."""
         for child in self.prepended_children:
             child.copyto(target)
-        target.write(self.stream.getvalue())
+        if self.stream:
+            target.write(self.stream.getvalue())
 
     def write(self, what):
+        if not self.stream:
+            self.stream = StringIO()
         self.stream.write(what)
 
+    def commit(self):
+        # Save what we have written until now so that the buffer
+        # itself is empty -- this makes it ready for insertion
+        if self.stream:
+            self.prepended_children.append(StringIOTree(self.stream))
+            self.stream = None
+
+    def insert(self, iotree):
+        """
+        Insert a StringIOTree (and all of its contents) at this location.
+        Further writing to self appears after what is inserted.
+        """
+        self.commit()
+        self.prepended_children.append(iotree)
+
     def insertion_point(self):
+        """
+        Returns a new StringIOTree, which is left behind at the current position
+        (it what is written to the result will appear right before whatever is
+        next written to self).
+
+        Calling getvalue() or copyto() on the result will only return the
+        contents written to it.
+        """
         # Save what we have written until now
-        # (would it be more efficient to check with len(self.stream.getvalue())?
-        # leaving it out for now)
-        self.prepended_children.append(StringIOTree(self.stream))
+        # This is so that getvalue on the result doesn't include it.
+        self.commit()
         # Construct the new forked object to return
         other = StringIOTree()
         self.prepended_children.append(other)
-        self.stream = StringIO()
         return other
 
 __doc__ = r"""
@@ -65,12 +88,16 @@ beta
 gamma
 <BLANKLINE>
 
+>>> i = StringIOTree()
+>>> d.insert(i)
+>>> i.write('inserted\n')
 >>> out = StringIO()
 >>> a.copyto(out)
 >>> print out.getvalue()
 first
 second
 alpha
+inserted
 beta
 gamma
 third