Refactor dependencies between buffer/memoryview utility code to avoid unused C...

Refactor dependencies between buffer/memoryview utility code to avoid unused C code and remove explicit utility code dependency lists from Python code where possible since they are much better expressed right in the C code.

Cython/Compiler/Buffer.py

@@ -36,7 +36,6 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
         if self.buffers_exists:
             use_bufstruct_declare_code(node.scope)
             use_py2_buffer_functions(node.scope)
-            node.scope.use_utility_code(empty_bufstruct_utility)
 
         return result
 
@@ -317,8 +316,8 @@ def put_init_vars(entry, code):
     code.putln("%s.data = NULL;" % pybuffernd_struct)
     code.putln("%s.rcbuffer = &%s;" % (pybuffernd_struct, pybuffer_struct))
 
+
 def put_acquire_arg_buffer(entry, code, pos):
-    code.globalstate.use_utility_code(acquire_utility_code)
     buffer_aux = entry.buffer_aux
     getbuffer = get_getbuffer_call(code, entry.cname, buffer_aux, entry.type)
 
@@ -331,10 +330,12 @@ def put_acquire_arg_buffer(entry, code, pos):
     # need to care about the buffer then.
     put_unpack_buffer_aux_into_scope(entry, code)
 
+
 def put_release_buffer_code(code, entry):
     code.globalstate.use_utility_code(acquire_utility_code)
     code.putln("__Pyx_SafeReleaseBuffer(&%s.rcbuffer->pybuffer);" % entry.buffer_aux.buflocal_nd_var.cname)
 
+
 def get_getbuffer_call(code, obj_cname, buffer_aux, buffer_type):
     ndim = buffer_type.ndim
     cast = int(buffer_type.cast)
@@ -343,10 +344,12 @@ def get_getbuffer_call(code, obj_cname, buffer_aux, buffer_type):
 
     dtype_typeinfo = get_type_information_cname(code, buffer_type.dtype)
 
+    code.globalstate.use_utility_code(acquire_utility_code)
     return ("__Pyx_GetBufferAndValidate(&%(pybuffernd_struct)s.rcbuffer->pybuffer, "
             "(PyObject*)%(obj_cname)s, &%(dtype_typeinfo)s, %(flags)s, %(ndim)d, "
             "%(cast)d, __pyx_stack)" % locals())
 
+
 def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry,
                          is_initialized, pos, code):
     """
@@ -364,7 +367,6 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry,
     """
 
     buffer_aux, buffer_type = buf_entry.buffer_aux, buf_entry.type
-    code.globalstate.use_utility_code(acquire_utility_code)
     pybuffernd_struct = buffer_aux.buflocal_nd_var.cname
     flags = get_flags(buffer_aux, buffer_type)
 
@@ -490,15 +492,6 @@ def use_bufstruct_declare_code(env):
     env.use_utility_code(buffer_struct_declare_code)
 
 
-def get_empty_bufstruct_code(max_ndim):
-    code = dedent("""
-        static Py_ssize_t __Pyx_zeros[] = {%s};
-        static Py_ssize_t __Pyx_minusones[] = {%s};
-    """) % (", ".join(["0"] * max_ndim), ", ".join(["-1"] * max_ndim))
-    return UtilityCode(proto=code)
-
-empty_bufstruct_utility = get_empty_bufstruct_code(Options.buffer_max_dims)
-
 def buf_lookup_full_code(proto, defin, name, nd):
     """
     Generates a buffer lookup function for the right number
@@ -519,6 +512,7 @@ def buf_lookup_full_code(proto, defin, name, nd):
         """) % (i, i, i, i) for i in range(nd)]
         ) + "\nreturn ptr;\n}")
 
+
 def buf_lookup_strided_code(proto, defin, name, nd):
     """
     Generates a buffer lookup function for the right number
@@ -529,6 +523,7 @@ def buf_lookup_strided_code(proto, defin, name, nd):
     offset = " + ".join(["i%d * s%d" % (i, i) for i in range(nd)])
     proto.putln("#define %s(type, buf, %s) (type)((char*)buf + %s)" % (name, args, offset))
 
+
 def buf_lookup_c_code(proto, defin, name, nd):
     """
     Similar to strided lookup, but can assume that the last dimension
@@ -542,6 +537,7 @@ def buf_lookup_c_code(proto, defin, name, nd):
         offset = " + ".join(["i%d * s%d" % (i, i) for i in range(nd - 1)])
         proto.putln("#define %s(type, buf, %s) ((type)((char*)buf + %s) + i%d)" % (name, args, offset, nd - 1))
 
+
 def buf_lookup_fortran_code(proto, defin, name, nd):
     """
     Like C lookup, but the first index is optimized instead.
@@ -557,6 +553,7 @@ def buf_lookup_fortran_code(proto, defin, name, nd):
 def use_py2_buffer_functions(env):
     env.use_utility_code(GetAndReleaseBufferUtilityCode())
 
+
 class GetAndReleaseBufferUtilityCode(object):
     # Emulation of PyObject_GetBuffer and PyBuffer_Release for Python 2.
     # For >= 2.6 we do double mode -- use the new buffer interface on objects
@@ -724,26 +721,18 @@ def load_buffer_utility(util_code_name, context=None, **kwargs):
     else:
         return TempitaUtilityCode.load(util_code_name, "Buffer.c", context=context, **kwargs)
 
-context = dict(max_dims=str(Options.buffer_max_dims))
-buffer_struct_declare_code = load_buffer_utility("BufferStructDeclare",
-                                                 context=context)
-
+context = dict(max_dims=Options.buffer_max_dims)
+buffer_struct_declare_code = load_buffer_utility("BufferStructDeclare", context=context)
+buffer_formats_declare_code = load_buffer_utility("BufferFormatStructs")
 
 # Utility function to set the right exception
 # The caller should immediately goto_error
 raise_indexerror_code = load_buffer_utility("BufferIndexError")
 raise_indexerror_nogil = load_buffer_utility("BufferIndexErrorNogil")
-
 raise_buffer_fallback_code = load_buffer_utility("BufferFallbackError")
-buffer_structs_code = load_buffer_utility(
-        "BufferFormatStructs", proto_block='utility_code_proto_before_types')
-acquire_utility_code = load_buffer_utility("BufferFormatCheck",
-                                           context=context,
-                                           requires=[buffer_structs_code,
-                                                     UtilityCode.load_cached("IsLittleEndian", "ModuleSetupCode.c")])
+
+acquire_utility_code = load_buffer_utility("BufferGetAndValidate", context=context)
+buffer_format_check_code = load_buffer_utility("BufferFormatCheck", context=context)
 
 # See utility code BufferFormatFromTypeInfo
-_typeinfo_to_format_code = load_buffer_utility("TypeInfoToFormat", context={},
-                                               requires=[buffer_structs_code])
-typeinfo_compare_code = load_buffer_utility("TypeInfoCompare", context={},
-                                            requires=[buffer_structs_code])
+_typeinfo_to_format_code = load_buffer_utility("TypeInfoToFormat")

Cython/Compiler/MemoryView.py

@@ -809,18 +809,15 @@ context = {
 }
 memviewslice_declare_code = load_memview_c_utility(
         "MemviewSliceStruct",
-        proto_block='utility_code_proto_before_types',
         context=context,
         requires=[])
 
-atomic_utility = load_memview_c_utility("Atomics", context,
-              proto_block='utility_code_proto_before_types')
+atomic_utility = load_memview_c_utility("Atomics", context)
 
 memviewslice_init_code = load_memview_c_utility(
     "MemviewSliceInit",
     context=dict(context, BUF_MAX_NDIMS=Options.buffer_max_dims),
     requires=[memviewslice_declare_code,
-              Buffer.acquire_utility_code,
               atomic_utility],
 )
 
@@ -842,7 +839,7 @@ view_utility_code = load_memview_cy_utility(
         context=context,
         requires=[Buffer.GetAndReleaseBufferUtilityCode(),
                   Buffer.buffer_struct_declare_code,
-                  Buffer.empty_bufstruct_utility,
+                  Buffer.buffer_formats_declare_code,
                   memviewslice_init_code,
                   is_contig_utility,
                   overlapping_utility,

Cython/Compiler/PyrexTypes.py

@@ -856,7 +856,6 @@ class MemoryViewSliceType(PyrexType):
             return TempitaUtilityCode.load(
                 "ObjectToMemviewSlice", "MemoryView_C.c", context=context)
 
-        env.use_utility_code(Buffer.acquire_utility_code)
         env.use_utility_code(MemoryView.memviewslice_init_code)
         env.use_utility_code(LazyUtilityCode(lazy_utility_callback))
 

Cython/Utility/Buffer.c

@@ -52,6 +52,7 @@ static void __Pyx_RaiseBufferFallbackError(void) {
 }
 
 /////////////// BufferFormatStructs.proto ///////////////
+//@proto_block: utility_code_proto_before_types
 
 #define IS_UNSIGNED(type) (((type) -1) > 0)
 
@@ -95,7 +96,9 @@ typedef struct {
   char is_valid_array;
 } __Pyx_BufFmt_Context;
 
+
 /////////////// GetAndReleaseBuffer.proto ///////////////
+
 #if PY_MAJOR_VERSION < 3
     static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags);
     static void __Pyx_ReleaseBuffer(Py_buffer *view);
@@ -105,6 +108,7 @@ typedef struct {
 #endif
 
 /////////////// GetAndReleaseBuffer ///////////////
+
 #if PY_MAJOR_VERSION < 3
 static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {
     if (PyObject_CheckBuffer(obj)) return PyObject_GetBuffer(obj, view, flags);
@@ -141,21 +145,8 @@ static void __Pyx_ReleaseBuffer(Py_buffer *view) {
 
 #endif /*  PY_MAJOR_VERSION < 3 */
 
-/////////////// BufferFormatCheck.proto ///////////////
-{{#
-
-    Buffer format string checking
-
-    Buffer type checking. Utility code for checking that acquired
-    buffers match our assumptions. We only need to check ndim and
-    the format string; the access mode/flags is checked by the
-    exporter. See:
 
-    http://docs.python.org/3/library/struct.html
-    http://legacy.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax
-
-    The alignment code is copied from _struct.c in Python.
-}}
+/////////////// BufferGetAndValidate.proto ///////////////
 
 #define __Pyx_GetBufferAndValidate(buf, obj, dtype, flags, nd, cast, stack) \
     ((obj == Py_None || obj == NULL) ? \
@@ -164,15 +155,88 @@ static void __Pyx_ReleaseBuffer(Py_buffer *view) {
 
 static int  __Pyx__GetBufferAndValidate(Py_buffer* buf, PyObject* obj,
     __Pyx_TypeInfo* dtype, int flags, int nd, int cast, __Pyx_BufFmt_StackElem* stack);
-static CYTHON_INLINE void __Pyx_ZeroBuffer(Py_buffer* buf);
-static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info);
+static void __Pyx_ZeroBuffer(Py_buffer* buf);
+static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info);/*proto*/
+
+static Py_ssize_t __Pyx_minusones[] = { {{ ", ".join(["-1"] * max_dims) }} };
+static Py_ssize_t __Pyx_zeros[] = { {{ ", ".join(["0"] * max_dims) }} };
+
+
+/////////////// BufferGetAndValidate ///////////////
+//@requires: BufferFormatCheck
+
+static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info) {
+  if (unlikely(info->buf == NULL)) return;
+  if (info->suboffsets == __Pyx_minusones) info->suboffsets = NULL;
+  __Pyx_ReleaseBuffer(info);
+}
+
+static void __Pyx_ZeroBuffer(Py_buffer* buf) {
+  buf->buf = NULL;
+  buf->obj = NULL;
+  buf->strides = __Pyx_zeros;
+  buf->shape = __Pyx_zeros;
+  buf->suboffsets = __Pyx_minusones;
+}
+
+static int __Pyx__GetBufferAndValidate(
+        Py_buffer* buf, PyObject* obj,  __Pyx_TypeInfo* dtype, int flags,
+        int nd, int cast, __Pyx_BufFmt_StackElem* stack)
+{
+  buf->buf = NULL;
+  if (unlikely(__Pyx_GetBuffer(obj, buf, flags) == -1)) {
+    __Pyx_ZeroBuffer(buf);
+    return -1;
+  }
+  // From this point on, we have acquired the buffer and must release it on errors.
+  if (unlikely(buf->ndim != nd)) {
+    PyErr_Format(PyExc_ValueError,
+                 "Buffer has wrong number of dimensions (expected %d, got %d)",
+                 nd, buf->ndim);
+    goto fail;
+  }
+  if (!cast) {
+    __Pyx_BufFmt_Context ctx;
+    __Pyx_BufFmt_Init(&ctx, stack, dtype);
+    if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail;
+  }
+  if (unlikely((unsigned)buf->itemsize != dtype->size)) {
+    PyErr_Format(PyExc_ValueError,
+      "Item size of buffer (%" CYTHON_FORMAT_SSIZE_T "d byte%s) does not match size of '%s' (%" CYTHON_FORMAT_SSIZE_T "d byte%s)",
+      buf->itemsize, (buf->itemsize > 1) ? "s" : "",
+      dtype->name, (Py_ssize_t)dtype->size, (dtype->size > 1) ? "s" : "");
+    goto fail;
+  }
+  if (buf->suboffsets == NULL) buf->suboffsets = __Pyx_minusones;
+  return 0;
+fail:;
+  __Pyx_SafeReleaseBuffer(buf);
+  return -1;
+}
+
+
+/////////////// BufferFormatCheck.proto ///////////////
+
+//  Buffer format string checking
+//
+//  Buffer type checking. Utility code for checking that acquired
+//  buffers match our assumptions. We only need to check ndim and
+//  the format string; the access mode/flags is checked by the
+//  exporter. See:
+//
+//  http://docs.python.org/3/library/struct.html
+//  http://legacy.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax
+//
+//  The alignment code is copied from _struct.c in Python.
+
 static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts);
 static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx,
                               __Pyx_BufFmt_StackElem* stack,
-                              __Pyx_TypeInfo* type); /* PROTO */
-
+                              __Pyx_TypeInfo* type); /*proto*/
 
 /////////////// BufferFormatCheck ///////////////
+//@requires: ModuleSetupCode.c::IsLittleEndian
+//@requires: BufferFormatStructs
 
 static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx,
                               __Pyx_BufFmt_StackElem* stack,
@@ -533,7 +597,7 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
 }
 
 /* Parse an array in the format string (e.g. (1,2,3)) */
-static CYTHON_INLINE PyObject *
+static PyObject *
 __pyx_buffmt_parse_array(__Pyx_BufFmt_Context* ctx, const char** tsp)
 {
     const char *ts = *tsp;
@@ -731,60 +795,13 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
   }
 }
 
-static CYTHON_INLINE void __Pyx_ZeroBuffer(Py_buffer* buf) {
-  buf->buf = NULL;
-  buf->obj = NULL;
-  buf->strides = __Pyx_zeros;
-  buf->shape = __Pyx_zeros;
-  buf->suboffsets = __Pyx_minusones;
-}
-
-static int __Pyx__GetBufferAndValidate(
-        Py_buffer* buf, PyObject* obj,  __Pyx_TypeInfo* dtype, int flags,
-        int nd, int cast, __Pyx_BufFmt_StackElem* stack)
-{
-  buf->buf = NULL;
-  if (unlikely(__Pyx_GetBuffer(obj, buf, flags) == -1)) {
-    __Pyx_ZeroBuffer(buf);
-    return -1;
-  }
-  // From this point on, we have acquired the buffer and must release it on errors.
-  if (unlikely(buf->ndim != nd)) {
-    PyErr_Format(PyExc_ValueError,
-                 "Buffer has wrong number of dimensions (expected %d, got %d)",
-                 nd, buf->ndim);
-    goto fail;
-  }
-  if (!cast) {
-    __Pyx_BufFmt_Context ctx;
-    __Pyx_BufFmt_Init(&ctx, stack, dtype);
-    if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail;
-  }
-  if (unlikely((unsigned)buf->itemsize != dtype->size)) {
-    PyErr_Format(PyExc_ValueError,
-      "Item size of buffer (%" CYTHON_FORMAT_SSIZE_T "d byte%s) does not match size of '%s' (%" CYTHON_FORMAT_SSIZE_T "d byte%s)",
-      buf->itemsize, (buf->itemsize > 1) ? "s" : "",
-      dtype->name, (Py_ssize_t)dtype->size, (dtype->size > 1) ? "s" : "");
-    goto fail;
-  }
-  if (buf->suboffsets == NULL) buf->suboffsets = __Pyx_minusones;
-  return 0;
-fail:;
-  __Pyx_SafeReleaseBuffer(buf);
-  return -1;
-}
-
-static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info) {
-  if (unlikely(info->buf == NULL)) return;
-  if (info->suboffsets == __Pyx_minusones) info->suboffsets = NULL;
-  __Pyx_ReleaseBuffer(info);
-}
-
 /////////////// TypeInfoCompare.proto ///////////////
 static int __pyx_typeinfo_cmp(__Pyx_TypeInfo *a, __Pyx_TypeInfo *b);
 
 /////////////// TypeInfoCompare ///////////////
-/* See if two dtypes are equal */
+//@requires: BufferFormatStructs
+
+// See if two dtypes are equal
 static int
 __pyx_typeinfo_cmp(__Pyx_TypeInfo *a, __Pyx_TypeInfo *b)
 {
@@ -843,7 +860,6 @@ __pyx_typeinfo_cmp(__Pyx_TypeInfo *a, __Pyx_TypeInfo *b)
 }
 
 
-
 /////////////// TypeInfoToFormat.proto ///////////////
 struct __pyx_typeinfo_string {
     char string[3];
@@ -851,7 +867,9 @@ struct __pyx_typeinfo_string {
 static struct __pyx_typeinfo_string __Pyx_TypeInfoToFormat(__Pyx_TypeInfo *type);
 
 /////////////// TypeInfoToFormat ///////////////
-{{# See also MemoryView.pyx:BufferFormatFromTypeInfo }}
+//@requires: BufferFormatStructs
+
+// See also MemoryView.pyx:BufferFormatFromTypeInfo
 
 static struct __pyx_typeinfo_string __Pyx_TypeInfoToFormat(__Pyx_TypeInfo *type) {
     struct __pyx_typeinfo_string result = { {0} };

Cython/Utility/MemoryView_C.c

 ////////// MemviewSliceStruct.proto //////////
+//@proto_block: utility_code_proto_before_types
 
 /* memoryview slice struct */
 struct {{memview_struct_name}};
@@ -16,6 +17,7 @@ typedef struct {
 
 
 /////////// Atomics.proto /////////////
+//@proto_block: utility_code_proto_before_types
 
 #include <pythread.h>
 
@@ -167,6 +169,8 @@ static int __Pyx_ValidateAndInit_memviewslice(
 
 /////////////// MemviewSliceValidateAndInit ///////////////
 //@requires: Buffer.c::TypeInfoCompare
+//@requires: Buffer.c::BufferFormatStructs
+//@requires: Buffer.c::BufferFormatCheck
 
 static int
 __pyx_check_strides(Py_buffer *buf, int dim, int ndim, int spec)