Utility Code loader + memview python3 compat

Cython/Compiler/Buffer.py

@@ -1090,9 +1090,6 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
 }
 
 static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) {
-  if (!ts) {
-    PyErr_SetString(PyExc_ValueError, "Got NULL buffer format");
-  }
   int got_Z = 0;
   while (1) {
     switch(*ts) {

Cython/Compiler/Code.pxd

 
 cimport cython
 
-cdef class UtilityCode:
+cdef class UtilityCodeBase(object):
+    pass
+
+cdef class UtilityCode(UtilityCodeBase):
     cdef public object proto
     cdef public object impl
     cdef public object init

Cython/Compiler/Code.py

@@ -8,7 +8,10 @@ cython.declare(re=object, Naming=object, Options=object, StringEncoding=object,
                Utils=object, SourceDescriptor=object, StringIOTree=object,
                DebugFlags=object, none_or_sub=object, basestring=object)
 
+import os
 import re
+import codecs
+
 import Naming
 import Options
 import StringEncoding
@@ -16,6 +19,7 @@ from Cython import Utils
 from Scanning import SourceDescriptor
 from Cython.StringIOTree import StringIOTree
 import DebugFlags
+import Errors
 
 from Cython.Utils import none_or_sub
 try:
@@ -38,17 +42,130 @@ uncachable_builtins = [
     'WindowsError',
     ]
 
-class UtilityCode(object):
+class UtilityCodeBase(object):
+
+    is_cython_utility = False
+
+    _utility_cache = {}
+
+    @classmethod
+    def _add_utility(self, utility, type, lines, begin_lineno):
+        if utility:
+            code = '\n' * begin_lineno + ''.join(lines)
+            if type == 'Proto':
+                utility[0] = code
+            else:
+                utility[1] = code
+
+    @classmethod
+    def load_utilities_from_file(cls, path):
+        utilities = cls._utility_cache.get(path)
+        if utilities:
+            return utilities
+
+        Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        filename = os.path.join(Cython_dir, "Utility", path)
+        f = codecs.open(filename, encoding='UTF-8')
+
+        _, ext = os.path.splitext(path)
+        if ext in ('.pyx', '.py', '.pxd', '.pxi'):
+            comment = '#'
+        else:
+            comment = '//'
+
+        regex = r'%s\s*Utility(Proto|Code)\s*:\s*((\w|\.)+)\s*' % comment
+        flags = re.DOTALL
+
+        utilities = {}
+        lines = []
+
+        utility = type = None
+        begin_lineno = 0
+        for lineno, line in enumerate(f):
+            m = re.search(regex, line)
+            if m:
+                cls._add_utility(utility, type, lines, begin_lineno)
+
+                begin_lineno = lineno + 1
+                type, name = m.group(1), m.group(2)
+                utility = utilities.setdefault(name, [None, None])
+                utilities[name] = utility
+
+                lines = []
+            else:
+                lines.append(line)
+
+        if not utility:
+            raise ValueError("Empty utility code file")
+
+        # Don't forget to add the last utility code
+        cls._add_utility(utility, type, lines, begin_lineno)
+
+        f.close()
+
+        cls._utility_cache[path] = utilities
+        return utilities
+
+    @classmethod
+    def load_utility_from_file(cls, path, util_code_name, proto_fmt_dict=None,
+                               impl_fmt_dict=None, *args, **kwargs):
+        """
+        Load a utility code from a file specified by path (relative to
+        Cython/Utility) and name util_code_name.
+
+        Utilities in the file can be specified as follows:
+
+            # UtilityProto: name
+            # UtilityImpl: name
+
+        for prototypes and implementation respectively. For non-python or
+        -cython files a // comment should be used instead.
+
+        proto_fmt_dict and impl_format_dict can optionally be given to perform
+        any substitutions.
+
+        If the @cname decorator is not used and this is a CythonUtilityCode,
+        one should pass in the 'name' keyword argument to be used for name
+        mangling of such entries.
+        """
+        proto, impl = cls.load_utility_as_string(path, util_code_name)
+
+        if proto:
+            if proto_fmt_dict:
+                proto = proto % proto_fmt_dict
+            kwargs['proto'] = proto
+
+        if impl:
+            if impl_fmt_dict:
+                impl = impl % impl_fmt_dict
+            kwargs['impl'] = impl
+
+        if 'name' not in kwargs:
+            kwargs['name'] = os.path.splitext(path)[0]
+
+        return cls(*args, **kwargs)
+
+    @classmethod
+    def load_utility_as_string(cls, path, util_code_name):
+        """
+        Load a utility code as a string. Returns (proto, implementation)
+        """
+        utilities = cls.load_utilities_from_file(path)
+        return utilities[util_code_name]
+
+    def __str__(self):
+        return "<%s(%s)" % (type(self).__name__, self.name)
+
+
+class UtilityCode(UtilityCodeBase):
     # Stores utility code to add during code generation.
     #
     # See GlobalState.put_utility_code.
     #
     # hashes/equals by instance
 
-    is_cython_utility = False
-
     def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
-                 proto_block='utility_code_proto'):
+                 proto_block='utility_code_proto', name=None):
         # proto_block: Which code block to dump prototype in. See GlobalState.
         self.proto = proto
         self.impl = impl
@@ -58,6 +175,7 @@ class UtilityCode(object):
         self._cache = {}
         self.specialize_list = []
         self.proto_block = proto_block
+        self.name = name
 
     def get_tree(self):
         pass

Cython/Compiler/CythonScope.py

@@ -5,6 +5,7 @@ from Errors import error
 from Scanning import StringSourceDescriptor
 import Options
 import Buffer
+import MemoryView
 
 class CythonScope(ModuleScope):
     is_cython_builtin = 1
@@ -100,11 +101,12 @@ def create_cython_scope(context, create_testscope):
     return scope
 
 
-cython_testscope_utility_code = CythonUtilityCode(u"""
-@cname('__pyx_testscope')
-cdef object _testscope(int value):
-    return "hello from cython scope, value=%d" % value
-""")
+# Load test utilities for the cython scope
+
+def load_testscope_utility(cython_util_name, *args, **kwargs):
+    return CythonUtilityCode.load_utility_from_file(
+                    "TestCythonScope.pyx", cython_util_name, *args, **kwargs)
+
 
 undecorated_methods_protos = UtilityCode(proto=u"""
     /* These methods are undecorated and have therefore no prototype */
@@ -116,237 +118,18 @@ undecorated_methods_protos = UtilityCode(proto=u"""
             PyObject *self, PyObject *value);
 """)
 
-test_cython_utility_dep = CythonUtilityCode(u"""
-@cname('__pyx_test_dep')
-cdef test_dep(obj):
-    print 'test_dep', obj
-""")
-
-cython_test_extclass_utility_code = CythonUtilityCode(
-        name="TestClassUtilityCode",
-        prefix="__pyx_prefix_TestClass_",
-        requires=[undecorated_methods_protos, test_cython_utility_dep],
-        impl=u"""
-cdef extern from *:
-    cdef object __pyx_test_dep(object)
-
-@cname('__pyx_TestClass')
-cdef class TestClass(object):
-    cdef public int value
-
-    def __init__(self, int value):
-        self.value = value
-
-    def __str__(self):
-        return 'TestClass(%d)' % self.value
-
-    cdef cdef_method(self, int value):
-        print 'Hello from cdef_method', value
-
-    cpdef cpdef_method(self, int value):
-        print 'Hello from cpdef_method', value
-
-    def def_method(self, int value):
-        print 'Hello from def_method', value
-
-    @cname('cdef_cname')
-    cdef cdef_cname_method(self, int value):
-        print "Hello from cdef_cname_method", value
-
-    @cname('cpdef_cname')
-    cpdef cpdef_cname_method(self, int value):
-        print "Hello from cpdef_cname_method", value
-
-    @cname('def_cname')
-    def def_cname_method(self, int value):
-        print "Hello from def_cname_method", value
-
-@cname('__pyx_test_call_other_cy_util')
-cdef test_call(obj):
-    print 'test_call'
-    __pyx_test_dep(obj)
-
-@cname('__pyx_TestClass_New')
-cdef _testclass_new(int value):
-    return TestClass(value)
-""")
-
-
-cythonview_testscope_utility_code = CythonUtilityCode(u"""
-@cname('__pyx_view_testscope')
-cdef object _testscope(int value):
-    return "hello from cython.view scope, value=%d" % value
-""")
-
-memview_name = u'memoryview'
-memview_typeptr_cname = '__pyx_memoryview_type'
-memview_objstruct_cname = '__pyx_memoryview_obj'
-view_utility_code = CythonUtilityCode(
-u"""
-cdef class Enum(object):
-    cdef object name
-    def __init__(self, name):
-        self.name = name
-    def __repr__(self):
-        return self.name
-
-cdef strided = Enum("<strided axis packing mode>")
-cdef contig = Enum("<contig axis packing mode>")
-cdef follow = Enum("<follow axis packing mode>")
-cdef direct = Enum("<direct axis access mode>")
-cdef ptr = Enum("<ptr axis access mode>")
-cdef full = Enum("<full axis access mode>")
-
-cdef extern from *:
-    int __Pyx_GetBuffer(object, Py_buffer *, int)
-    void __Pyx_ReleaseBuffer(Py_buffer *)
-
-@cname('__pyx_memoryview')
-cdef class memoryview(object):
-
-    cdef object obj
-    cdef Py_buffer view
-    cdef int gotbuf_flag
-
-    def __cinit__(memoryview self, object obj, int flags):
-        self.obj = obj
-        __Pyx_GetBuffer(self.obj, &self.view, flags)
-
-    def __dealloc__(memoryview self):
-        self.obj = None
-        __Pyx_ReleaseBuffer(&self.view)
-
-@cname('__pyx_memoryview_new')
-cdef memoryview memoryview_cwrapper(object o, int flags):
-    return memoryview(o, flags)
-""", name="view_code", requires=(Buffer.GetAndReleaseBufferUtilityCode(),))
-
-cython_array_utility_code = CythonUtilityCode(u'''
-cdef extern from "stdlib.h":
-    void *malloc(size_t)
-    void free(void *)
-
-cdef extern from "Python.h":
-
-    cdef enum:
-        PyBUF_C_CONTIGUOUS,
-        PyBUF_F_CONTIGUOUS,
-        PyBUF_ANY_CONTIGUOUS
-
-@cname("__pyx_array")
-cdef class array:
-
-    cdef:
-        char *data
-        Py_ssize_t len
-        char *format
-        int ndim
-        Py_ssize_t *shape
-        Py_ssize_t *strides
-        Py_ssize_t itemsize
-        str mode
-        void (*callback_free_data)(char *data)
-
-    def __cinit__(array self, tuple shape, Py_ssize_t itemsize, char *format,
-                  mode="c", bint allocate_buffer=True):
-
-        self.ndim = len(shape)
-        self.itemsize = itemsize
-
-        if not self.ndim:
-            raise ValueError("Empty shape tuple for cython.array")
-
-        if self.itemsize <= 0:
-            raise ValueError("itemsize <= 0 for cython.array")
-
-        self.format = format
-
-        self.shape = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
-        self.strides = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
-
-        if not self.shape or not self.strides:
-            raise MemoryError("unable to allocate shape or strides.")
-
-        cdef int idx
-        # cdef Py_ssize_t dim, stride
-        idx = 0
-        for dim in shape:
-            if dim <= 0:
-                raise ValueError("Invalid shape.")
-
-            self.shape[idx] = dim
-            idx += 1
-
-        stride = itemsize
-        if mode == "fortran":
-            idx = 0
-            for dim in shape:
-                self.strides[idx] = stride
-                stride = stride * dim
-                idx += 1
-        elif mode == "c":
-            idx = self.ndim-1
-            for dim in shape[::-1]:
-                self.strides[idx] = stride
-                stride = stride * dim
-                idx -= 1
-        else:
-            raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode)
-
-        self.len = stride
-        self.mode = mode
-
-        if allocate_buffer:
-            self.data = <char *>malloc(self.len)
-            if not self.data:
-                raise MemoryError("unable to allocate array data.")
-        else:
-            self.data = NULL
-
-        self.callback_free_data = NULL
+cython_testscope_utility_code = load_testscope_utility("TestScope")
 
-    def __getbuffer__(self, Py_buffer *info, int flags):
+test_cython_utility_dep = load_testscope_utility("TestDep")
 
-        cdef int bufmode = -1
-        if self.mode == b"c":
-            bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
-        elif self.mode == b"fortran":
-            bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
-        if not (flags & bufmode):
-            raise ValueError("Can only create a buffer that is contiguous in memory.")
-        info.buf = self.data
-        info.len = self.len
-        info.ndim = self.ndim
-        info.shape = self.shape
-        info.strides = self.strides
-        info.suboffsets = NULL
-        info.itemsize = self.itemsize
-        info.format = self.format
-        # we do not need to call releasebuffer
-        info.obj = None
+cython_test_extclass_utility_code = \
+    load_testscope_utility("TestClass", name="TestClass",
+                           requires=[undecorated_methods_protos,
+                                     test_cython_utility_dep])
 
-    def __releasebuffer__(array self, Py_buffer* info):
-        # array.__releasebuffer__ should not be called,
-        # because the Py_buffer's 'obj' field is set to None.
-        raise NotImplementedError()
+cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
 
-    def __dealloc__(array self):
-        if self.data:
-            if self.callback_free_data != NULL:
-                self.callback_free_data(self.data)
-            else:
-                free(self.data)
-            self.data = NULL
-        if self.strides:
-            free(self.strides)
-            self.strides = NULL
-        if self.shape:
-            free(self.shape)
-            self.shape = NULL
-        self.format = NULL
-        self.itemsize = 0
+view_utility_code = MemoryView.load_memview_cy_utility(
+        "MemoryView", requires=(Buffer.GetAndReleaseBufferUtilityCode(),))
 
-@cname("__pyx_array_new")
-cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode):
-    return array(shape, itemsize, format, mode)
-''')
+cython_array_utility_code = MemoryView.load_memview_cy_utility("CythonArray")

Cython/Compiler/MemoryView.py

@@ -2,7 +2,6 @@ from Errors import CompileError
 from ExprNodes import IntNode, NoneNode, IntBinopNode, NameNode, AttributeNode
 from Visitor import CythonTransform
 import Options
-import CythonScope
 from Code import UtilityCode
 from UtilityCode import CythonUtilityCode
 from PyrexTypes import py_object_type, cython_memoryview_ptr_type
@@ -47,6 +46,10 @@ _spec_to_const = {
         'full'   : MEMVIEW_FULL
         }
 
+memview_name = u'memoryview'
+memview_typeptr_cname = '__pyx_memoryview_type'
+memview_objstruct_cname = '__pyx_memoryview_obj'
+memviewslice_cname = u'__Pyx_memviewslice'
 
 def specs_to_code(specs):
     arr = []
@@ -694,309 +697,33 @@ class MemoryViewSliceTransform(CythonTransform):
     def visit_SingleAssignmentNode(self, node):
         return node
 
-copy_template = '''
-static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
-
-    __Pyx_RefNannyDeclarations
-    int i;
-    __Pyx_memviewslice new_mvs = {0, 0};
-    struct __pyx_memoryview_obj *from_memview = from_mvs.memview;
-    Py_buffer *buf = &from_memview->view;
-    PyObject *shape_tuple = 0;
-    PyObject *temp_int = 0;
-    struct __pyx_array_obj *array_obj = 0;
-    struct __pyx_memoryview_obj *memview_obj = 0;
-    char *mode = "%(mode)s";
-
-    __Pyx_RefNannySetupContext("%(copy_name)s");
-
-    shape_tuple = PyTuple_New((Py_ssize_t)(buf->ndim));
-    if(unlikely(!shape_tuple)) {
-        goto fail;
-    }
-    __Pyx_GOTREF(shape_tuple);
-
-
-    for(i=0; i<buf->ndim; i++) {
-        temp_int = PyInt_FromLong(buf->shape[i]);
-        if(unlikely(!temp_int)) {
-            goto fail;
-        } else {
-            PyTuple_SET_ITEM(shape_tuple, i, temp_int);
-        }
-    }
-
-    array_obj = __pyx_array_new(shape_tuple, %(sizeof_dtype)s, buf->format, mode);
-    if (unlikely(!array_obj)) {
-        goto fail;
-    }
-    __Pyx_GOTREF(array_obj);
 
-    memview_obj = __pyx_memoryview_new((PyObject *) array_obj, %(contig_flag)s);
-    if (unlikely(!memview_obj)) {
-        goto fail;
-    }
+def load_memview_cy_utility(name, *args, **kwargs):
+    return CythonUtilityCode.load_utility_from_file(
+                        "MemoryView.pyx", name, *args, **kwargs)
 
-    /* initialize new_mvs */
-    if (unlikely(-1 == __Pyx_init_memviewslice(memview_obj, buf->ndim, &new_mvs))) {
-        PyErr_SetString(PyExc_RuntimeError,
-            "Could not initialize new memoryviewslice object.");
-        goto fail;
-    }
+def load_memview_c_utility(name, *args, **kwargs):
+    return UtilityCode.load_utility_from_file(
+                        "MemoryView.c", name, *args, **kwargs)
 
-    if (unlikely(-1 == %(copy_contents_name)s(&from_mvs, &new_mvs))) {
-        /* PyErr_SetString(PyExc_RuntimeError,
-            "Could not copy contents of memoryview slice."); */
-        goto fail;
-    }
+def load_memview_c_string(name):
+    return UtilityCode.load_utility_as_string("MemoryView.c", name)
 
-    goto no_fail;
-
-fail:
-    __Pyx_XDECREF(new_mvs.memview); new_mvs.memview = 0;
-    new_mvs.data = 0;
-no_fail:
-    __Pyx_XDECREF(shape_tuple); shape_tuple = 0;
-    __Pyx_GOTREF(temp_int);
-    __Pyx_XDECREF(temp_int); temp_int = 0;
-    __Pyx_XDECREF(array_obj); array_obj = 0;
-    __Pyx_RefNannyFinishContext();
-    return new_mvs;
+_, copy_template = UtilityCode.load_utility_as_string(
+                "MemoryView.c", "MemviewSliceCopyTemplate")
 
+fmt_dict = {
+    'memview_struct_name': memview_objstruct_cname,
+    'max_dims': Options.buffer_max_dims,
+    'memviewslice_name': memviewslice_cname,
 }
-'''
-
-spec_constants_code = UtilityCode(proto="""
-#define __Pyx_MEMVIEW_DIRECT  1
-#define __Pyx_MEMVIEW_PTR     2
-#define __Pyx_MEMVIEW_FULL    4
-#define __Pyx_MEMVIEW_CONTIG  8
-#define __Pyx_MEMVIEW_STRIDED 16
-#define __Pyx_MEMVIEW_FOLLOW  32
-"""
-)
-
-memviewslice_cname = u'__Pyx_memviewslice'
-memviewslice_declare_code = UtilityCode(
+memviewslice_declare_code = load_memview_c_utility(
+        name="MemviewSliceStruct",
         proto_block='utility_code_proto_before_types',
-        proto="""
-
-/* memoryview slice struct */
+        proto_fmt_dict=fmt_dict)
 
-typedef struct {
-  struct %(memview_struct_name)s *memview;
-  char *data;
-  Py_ssize_t shape[%(max_dims)d];
-  Py_ssize_t strides[%(max_dims)d];
-  Py_ssize_t suboffsets[%(max_dims)d];
-} %(memviewslice_name)s;
-
-""" % dict(memview_struct_name=CythonScope.memview_objstruct_cname,
-            max_dims=Options.buffer_max_dims,
-            memviewslice_name=memviewslice_cname)
+memviewslice_init_code = load_memview_c_utility(
+    name="MemviewSliceInit",
+    proto_fmt_dict={'BUF_MAX_NDIMS': Options.buffer_max_dims},
+    requires=[memviewslice_declare_code],
 )
-
-memviewslice_init_code = UtilityCode(proto="""\
-
-#define __Pyx_BUF_MAX_NDIMS %(BUF_MAX_NDIMS)d
-
-#define __Pyx_MEMVIEW_DIRECT   1
-#define __Pyx_MEMVIEW_PTR      2
-#define __Pyx_MEMVIEW_FULL     4
-#define __Pyx_MEMVIEW_CONTIG   8
-#define __Pyx_MEMVIEW_STRIDED  16
-#define __Pyx_MEMVIEW_FOLLOW   32
-
-#define __Pyx_IS_C_CONTIG 1
-#define __Pyx_IS_F_CONTIG 2
-
-static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview,
-                                int *axes_specs, int c_or_f_flag,  int ndim, __Pyx_TypeInfo *dtype,
-                                __Pyx_BufFmt_StackElem stack[], __Pyx_memviewslice *memviewslice);
-
-static int __Pyx_init_memviewslice(
-                struct __pyx_memoryview_obj *memview,
-                int ndim,
-                __Pyx_memviewslice *memviewslice);
-""" % {'BUF_MAX_NDIMS' :Options.buffer_max_dims},
-impl = """\
-static int __Pyx_ValidateAndInit_memviewslice(
-                struct __pyx_memoryview_obj *memview,
-                int *axes_specs,
-                int c_or_f_flag,
-                int ndim,
-                __Pyx_TypeInfo *dtype, 
-                __Pyx_BufFmt_StackElem stack[],
-                __Pyx_memviewslice *memviewslice) {
-
-    __Pyx_RefNannyDeclarations
-    __Pyx_RefNannySetupContext("ValidateAndInit_memviewslice");
-    Py_buffer *buf = &memview->view;
-    int stride, i, spec = 0, retval = -1;
-
-    if (!buf) goto fail;
-
-    if(memviewslice->data || memviewslice->memview) {
-        PyErr_SetString(PyExc_ValueError,
-            "memoryviewslice struct must be initialized to NULL.");
-        goto fail;
-    }
-
-    if (buf->ndim != ndim) {
-        PyErr_Format(PyExc_ValueError,
-                "Buffer has wrong number of dimensions (expected %d, got %d)",
-                ndim, buf->ndim);
-        goto fail;
-    }
-
-    __Pyx_BufFmt_Context ctx;
-    __Pyx_BufFmt_Init(&ctx, stack, dtype);
-    if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail;
-
-    if ((unsigned)buf->itemsize != dtype->size) {
-        PyErr_Format(PyExc_ValueError,
-          "Item size of buffer (%"PY_FORMAT_SIZE_T"d byte%s) does not match size of '%s' (%"PY_FORMAT_SIZE_T"d byte%s)",
-          buf->itemsize, (buf->itemsize > 1) ? "s" : "",
-          dtype->name,
-          dtype->size, (dtype->size > 1) ? "s" : "");
-        goto fail;
-    }
-
-    if (!buf->strides) {
-        PyErr_SetString(PyExc_ValueError,
-            "buffer does not supply strides necessary for memoryview.");
-        goto fail;
-    }
-
-    for(i=0; i<ndim; i++) {
-        spec = axes_specs[i];
-
-        if (spec & __Pyx_MEMVIEW_CONTIG) {
-            if (buf->strides[i] != buf->itemsize) {
-                PyErr_SetString(PyExc_ValueError,
-                    "Buffer and memoryview are not contiguous in the same dimension.");
-                goto fail;
-            }
-        }
-
-        if (spec & (__Pyx_MEMVIEW_STRIDED | __Pyx_MEMVIEW_FOLLOW)) {
-            if (buf->strides[i] < buf->itemsize) {
-                PyErr_SetString(PyExc_ValueError,
-                    "Buffer and memoryview are not contiguous in the same dimension.");
-                goto fail;
-            }
-        }
-
-        if (spec & __Pyx_MEMVIEW_DIRECT) {
-            if (buf->suboffsets && buf->suboffsets[i] >= 0) {
-                PyErr_SetString(PyExc_ValueError,
-                    "Buffer not compatible with direct access.");
-                goto fail;
-            }
-        }
-
-        if (spec & (__Pyx_MEMVIEW_PTR | __Pyx_MEMVIEW_FULL)) {
-            if (!buf->suboffsets) {
-                PyErr_SetString(PyExc_ValueError,
-                    "Buffer not able to be indirectly accessed.");
-                goto fail;
-            }
-        }
-
-        if (spec & __Pyx_MEMVIEW_PTR) {
-            if (buf->suboffsets[i] < 0) {
-                PyErr_Format(PyExc_ValueError,
-                    "Buffer not indirectly accessed in %d dimension, although memoryview is.", i);
-                goto fail;
-            }
-        }
-    }
-
-    if (c_or_f_flag & __Pyx_IS_F_CONTIG) {
-        stride = 1;
-        for(i=0; i<ndim; i++) {
-            if(stride * buf->itemsize != buf->strides[i]) {
-                PyErr_SetString(PyExc_ValueError,
-                    "Buffer not fortran contiguous.");
-                goto fail;
-            }
-            stride = stride * buf->shape[i];
-        }
-    } else if (c_or_f_flag & __Pyx_IS_F_CONTIG) {
-        for(i=ndim-1; i>-1; i--) {
-            if(stride * buf->itemsize != buf->strides[i]) {
-                PyErr_SetString(PyExc_ValueError,
-                    "Buffer not C contiguous.");
-                goto fail;
-            }
-            stride = stride * buf->shape[i];
-        }
-    }
-
-    if(unlikely(__Pyx_init_memviewslice(memview, ndim, memviewslice) == -1)) {
-        goto fail;
-    }
-
-    retval = 0;
-    goto no_fail;
-fail:
-    __Pyx_XDECREF(memviewslice->memview);
-    memviewslice->memview = 0;
-    memviewslice->data = 0;
-    retval = -1;
-
-no_fail:
-    __Pyx_RefNannyFinishContext();
-    return retval;
-}
-
-static int __Pyx_init_memviewslice(
-                struct __pyx_memoryview_obj *memview,
-                int ndim,
-                __Pyx_memviewslice *memviewslice) {
-
-    __Pyx_RefNannyDeclarations
-    __Pyx_RefNannySetupContext("init_memviewslice");
-    int i, retval=-1;
-    Py_buffer *buf = &memview->view;
-
-    if(!buf) {
-        PyErr_SetString(PyExc_ValueError,
-            "buf is NULL.");
-        goto fail;
-    } else if(memviewslice->memview || memviewslice->data) {
-        PyErr_SetString(PyExc_ValueError,
-            "memviewslice is already initialized!");
-        goto fail;
-    }
-
-    for(i=0; i<ndim; i++) {
-        memviewslice->strides[i] = buf->strides[i];
-        memviewslice->shape[i]   = buf->shape[i];
-        if(buf->suboffsets) {
-            memviewslice->suboffsets[i] = buf->suboffsets[i];
-        } else {
-            memviewslice->suboffsets[i] = -1;
-        }
-    }
-
-    __Pyx_INCREF((PyObject *)memview);
-    __Pyx_GIVEREF((PyObject *)memview);
-    memviewslice->memview = memview;
-    memviewslice->data = (char *)buf->buf;
-    retval = 0;
-    goto no_fail;
-
-fail:
-    __Pyx_XDECREF(memviewslice->memview);
-    memviewslice->memview = 0;
-    memviewslice->data = 0;
-    retval = -1;
-no_fail:
-    __Pyx_RefNannyFinishContext();
-    return retval;
-}
-""")
-
-memviewslice_init_code.requires = [memviewslice_declare_code]
-

Cython/Compiler/PyrexTypes.py

@@ -498,6 +498,7 @@ class MemoryViewSliceType(PyrexType):
     def check_for_null_code(self, cname):
         return cname + '.memview'
 
+
 class BufferType(BaseType):
     #
     #  Delegates most attribute

Cython/Compiler/UtilityCode.py

@@ -3,6 +3,7 @@ from Scanning import StringSourceDescriptor
 import Symtab
 import Naming
 from Cython.Compiler import Visitor
+import Code
 
 class NonManglingModuleScope(Symtab.ModuleScope):
 
@@ -43,7 +44,7 @@ class CythonUtilityCodeContext(StringParseContext):
         return self.scope
 
 
-class CythonUtilityCode(object):
+class CythonUtilityCode(Code.UtilityCodeBase):
     """
     Utility code written in the Cython language itself.
 

Cython/Utility/MemoryView.c

+// UtilityProto: MemviewSliceStruct
+
+/* memoryview slice struct */
+
+typedef struct {
+  struct %(memview_struct_name)s *memview;
+  char *data;
+  Py_ssize_t shape[%(max_dims)d];
+  Py_ssize_t strides[%(max_dims)d];
+  Py_ssize_t suboffsets[%(max_dims)d];
+} %(memviewslice_name)s;
+
+// UtilityProto: MemviewSliceInit
+
+#define __Pyx_BUF_MAX_NDIMS %(BUF_MAX_NDIMS)d
+
+#define __Pyx_MEMVIEW_DIRECT   1
+#define __Pyx_MEMVIEW_PTR      2
+#define __Pyx_MEMVIEW_FULL     4
+#define __Pyx_MEMVIEW_CONTIG   8
+#define __Pyx_MEMVIEW_STRIDED  16
+#define __Pyx_MEMVIEW_FOLLOW   32
+
+#define __Pyx_IS_C_CONTIG 1
+#define __Pyx_IS_F_CONTIG 2
+
+static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview,
+                                int *axes_specs, int c_or_f_flag,  int ndim, __Pyx_TypeInfo *dtype,
+                                __Pyx_BufFmt_StackElem stack[], __Pyx_memviewslice *memviewslice);
+
+static int __Pyx_init_memviewslice(
+                struct __pyx_memoryview_obj *memview,
+                int ndim,
+                __Pyx_memviewslice *memviewslice);
+
+// UtilityCode: MemviewSliceInit
+
+static int __Pyx_ValidateAndInit_memviewslice(
+                struct __pyx_memoryview_obj *memview,
+                int *axes_specs,
+                int c_or_f_flag,
+                int ndim,
+                __Pyx_TypeInfo *dtype, 
+                __Pyx_BufFmt_StackElem stack[],
+                __Pyx_memviewslice *memviewslice) {
+
+    __Pyx_RefNannyDeclarations
+    __Pyx_RefNannySetupContext("ValidateAndInit_memviewslice");
+    Py_buffer *buf = &memview->view;
+    int stride, i, spec = 0, retval = -1;
+
+    if (!buf) goto fail;
+
+    if(memviewslice->data || memviewslice->memview) {
+        PyErr_SetString(PyExc_ValueError,
+            "memoryviewslice struct must be initialized to NULL.");
+        goto fail;
+    }
+
+    if (buf->ndim != ndim) {
+        PyErr_Format(PyExc_ValueError,
+                "Buffer has wrong number of dimensions (expected %d, got %d)",
+                ndim, buf->ndim);
+        goto fail;
+    }
+
+    __Pyx_BufFmt_Context ctx;
+    __Pyx_BufFmt_Init(&ctx, stack, dtype);
+    if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail;
+
+    if ((unsigned)buf->itemsize != dtype->size) {
+        PyErr_Format(PyExc_ValueError,
+                     "Item size of buffer (%" PY_FORMAT_SIZE_T "d byte%s) "
+                     "does not match size of '%s' (%" PY_FORMAT_SIZE_T "d byte%s)",
+                     buf->itemsize,
+                     (buf->itemsize > 1) ? "s" : "",
+                     dtype->name,
+                     dtype->size,
+                     (dtype->size > 1) ? "s" : "");
+        goto fail;
+    }
+
+    if (!buf->strides) {
+        PyErr_SetString(PyExc_ValueError,
+            "buffer does not supply strides necessary for memoryview.");
+        goto fail;
+    }
+
+    for(i=0; i<ndim; i++) {
+        spec = axes_specs[i];
+
+        if (spec & __Pyx_MEMVIEW_CONTIG) {
+            if (buf->strides[i] != buf->itemsize) {
+                PyErr_SetString(PyExc_ValueError,
+                    "Buffer and memoryview are not contiguous in the same dimension.");
+                goto fail;
+            }
+        }
+
+        if (spec & (__Pyx_MEMVIEW_STRIDED | __Pyx_MEMVIEW_FOLLOW)) {
+            if (buf->strides[i] < buf->itemsize) {
+                PyErr_SetString(PyExc_ValueError,
+                    "Buffer and memoryview are not contiguous in the same dimension.");
+                goto fail;
+            }
+        }
+
+        if (spec & __Pyx_MEMVIEW_DIRECT) {
+            if (buf->suboffsets && buf->suboffsets[i] >= 0) {
+                PyErr_SetString(PyExc_ValueError,
+                    "Buffer not compatible with direct access.");
+                goto fail;
+            }
+        }
+
+        if (spec & (__Pyx_MEMVIEW_PTR | __Pyx_MEMVIEW_FULL)) {
+            if (!buf->suboffsets) {
+                PyErr_SetString(PyExc_ValueError,
+                    "Buffer not able to be indirectly accessed.");
+                goto fail;
+            }
+        }
+
+        if (spec & __Pyx_MEMVIEW_PTR) {
+            if (buf->suboffsets[i] < 0) {
+                PyErr_Format(PyExc_ValueError,
+                    "Buffer not indirectly accessed in %d dimension, although memoryview is.", i);
+                goto fail;
+            }
+        }
+    }
+
+    if (c_or_f_flag & __Pyx_IS_F_CONTIG) {
+        stride = 1;
+        for(i=0; i<ndim; i++) {
+            if(stride * buf->itemsize != buf->strides[i]) {
+                PyErr_SetString(PyExc_ValueError,
+                    "Buffer not fortran contiguous.");
+                goto fail;
+            }
+            stride = stride * buf->shape[i];
+        }
+    } else if (c_or_f_flag & __Pyx_IS_F_CONTIG) {
+        for(i=ndim-1; i>-1; i--) {
+            if(stride * buf->itemsize != buf->strides[i]) {
+                PyErr_SetString(PyExc_ValueError,
+                    "Buffer not C contiguous.");
+                goto fail;
+            }
+            stride = stride * buf->shape[i];
+        }
+    }
+
+    if(unlikely(__Pyx_init_memviewslice(memview, ndim, memviewslice) == -1)) {
+        goto fail;
+    }
+
+    retval = 0;
+    goto no_fail;
+fail:
+    __Pyx_XDECREF(memviewslice->memview);
+    memviewslice->memview = 0;
+    memviewslice->data = 0;
+    retval = -1;
+
+no_fail:
+    __Pyx_RefNannyFinishContext();
+    return retval;
+}
+
+static int __Pyx_init_memviewslice(
+                struct __pyx_memoryview_obj *memview,
+                int ndim,
+                __Pyx_memviewslice *memviewslice) {
+
+    __Pyx_RefNannyDeclarations
+    __Pyx_RefNannySetupContext("init_memviewslice");
+    int i, retval=-1;
+    Py_buffer *buf = &memview->view;
+
+    if(!buf) {
+        PyErr_SetString(PyExc_ValueError,
+            "buf is NULL.");
+        goto fail;
+    } else if(memviewslice->memview || memviewslice->data) {
+        PyErr_SetString(PyExc_ValueError,
+            "memviewslice is already initialized!");
+        goto fail;
+    }
+
+    for(i=0; i<ndim; i++) {
+        memviewslice->strides[i] = buf->strides[i];
+        memviewslice->shape[i]   = buf->shape[i];
+        if(buf->suboffsets) {
+            memviewslice->suboffsets[i] = buf->suboffsets[i];
+        } else {
+            memviewslice->suboffsets[i] = -1;
+        }
+    }
+
+    __Pyx_INCREF((PyObject *)memview);
+    __Pyx_GIVEREF((PyObject *)memview);
+    memviewslice->memview = memview;
+    memviewslice->data = (char *)buf->buf;
+    retval = 0;
+    goto no_fail;
+
+fail:
+    __Pyx_XDECREF(memviewslice->memview);
+    memviewslice->memview = 0;
+    memviewslice->data = 0;
+    retval = -1;
+no_fail:
+    __Pyx_RefNannyFinishContext();
+    return retval;
+}
+
+// UtilityCode: MemviewSliceCopyTemplate
+
+static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
+
+    __Pyx_RefNannyDeclarations
+    int i;
+    __Pyx_memviewslice new_mvs = {0, 0};
+    struct __pyx_memoryview_obj *from_memview = from_mvs.memview;
+    Py_buffer *buf = &from_memview->view;
+    PyObject *shape_tuple = 0;
+    PyObject *temp_int = 0;
+    struct __pyx_array_obj *array_obj = 0;
+    struct __pyx_memoryview_obj *memview_obj = 0;
+    char *mode = "%(mode)s";
+
+    __Pyx_RefNannySetupContext("%(copy_name)s");
+
+    shape_tuple = PyTuple_New((Py_ssize_t)(buf->ndim));
+    if(unlikely(!shape_tuple)) {
+        goto fail;
+    }
+    __Pyx_GOTREF(shape_tuple);
+
+
+    for(i=0; i<buf->ndim; i++) {
+        temp_int = PyInt_FromLong(buf->shape[i]);
+        if(unlikely(!temp_int)) {
+            goto fail;
+        } else {
+            PyTuple_SET_ITEM(shape_tuple, i, temp_int);
+        }
+    }
+
+    array_obj = __pyx_array_new(shape_tuple, %(sizeof_dtype)s, buf->format, mode);
+    if (unlikely(!array_obj)) {
+        goto fail;
+    }
+    __Pyx_GOTREF(array_obj);
+
+    memview_obj = __pyx_memoryview_new((PyObject *) array_obj, %(contig_flag)s);
+    if (unlikely(!memview_obj)) {
+        goto fail;
+    }
+
+    /* initialize new_mvs */
+    if (unlikely(-1 == __Pyx_init_memviewslice(memview_obj, buf->ndim, &new_mvs))) {
+        PyErr_SetString(PyExc_RuntimeError,
+            "Could not initialize new memoryviewslice object.");
+        goto fail;
+    }
+
+    if (unlikely(-1 == %(copy_contents_name)s(&from_mvs, &new_mvs))) {
+        /* PyErr_SetString(PyExc_RuntimeError,
+            "Could not copy contents of memoryview slice."); */
+        goto fail;
+    }
+
+    goto no_fail;
+
+fail:
+    __Pyx_XDECREF(new_mvs.memview); new_mvs.memview = 0;
+    new_mvs.data = 0;
+no_fail:
+    __Pyx_XDECREF(shape_tuple); shape_tuple = 0;
+    __Pyx_GOTREF(temp_int);
+    __Pyx_XDECREF(temp_int); temp_int = 0;
+    __Pyx_XDECREF(array_obj); array_obj = 0;
+    __Pyx_RefNannyFinishContext();
+    return new_mvs;
+
+}
+

Cython/Utility/MemoryView.pyx

+# UtilityCode: CythonArray
+
+cdef extern from "stdlib.h":
+    void *malloc(size_t)
+    void free(void *)
+
+cdef extern from "Python.h":
+
+    cdef enum:
+        PyBUF_C_CONTIGUOUS,
+        PyBUF_F_CONTIGUOUS,
+        PyBUF_ANY_CONTIGUOUS
+
+@cname("__pyx_array")
+cdef class array:
+
+    cdef:
+        char *data
+        Py_ssize_t len
+        char *format
+        int ndim
+        Py_ssize_t *shape
+        Py_ssize_t *strides
+        Py_ssize_t itemsize
+        unicode mode
+        bytes _format
+        void (*callback_free_data)(char *data)
+
+    def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format,
+                  mode=u"c", bint allocate_buffer=True):
+
+        self.ndim = len(shape)
+        self.itemsize = itemsize
+
+        if not self.ndim:
+            raise ValueError("Empty shape tuple for cython.array")
+
+        if self.itemsize <= 0:
+            raise ValueError("itemsize <= 0 for cython.array")
+
+        encode = getattr(format, 'encode', None)
+        if encode:
+            format = encode('ASCII')
+        self._format = format
+        self.format = self._format
+
+        self.shape = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
+        self.strides = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
+
+        if not self.shape or not self.strides:
+            raise MemoryError("unable to allocate shape or strides.")
+
+        cdef int idx
+        # cdef Py_ssize_t dim, stride
+        idx = 0
+        for dim in shape:
+            if dim <= 0:
+                raise ValueError("Invalid shape.")
+
+            self.shape[idx] = dim
+            idx += 1
+
+        stride = itemsize
+        if mode == "fortran":
+            idx = 0
+            for dim in shape:
+                self.strides[idx] = stride
+                stride = stride * dim
+                idx += 1
+        elif mode == "c":
+            idx = self.ndim-1
+            for dim in shape[::-1]:
+                self.strides[idx] = stride
+                stride = stride * dim
+                idx -= 1
+        else:
+            raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode)
+
+        self.len = stride
+
+        decode = getattr(mode, 'decode', None)
+        if decode:
+            mode = decode('ASCII')
+        self.mode = mode
+
+        if allocate_buffer:
+            self.data = <char *>malloc(self.len)
+            if not self.data:
+                raise MemoryError("unable to allocate array data.")
+        else:
+            self.data = NULL
+
+        self.callback_free_data = NULL
+
+    def __getbuffer__(self, Py_buffer *info, int flags):
+
+        cdef int bufmode = -1
+        if self.mode == b"c":
+            bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
+        elif self.mode == b"fortran":
+            bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
+        if not (flags & bufmode):
+            raise ValueError("Can only create a buffer that is contiguous in memory.")
+        info.buf = self.data
+        info.len = self.len
+        info.ndim = self.ndim
+        info.shape = self.shape
+        info.strides = self.strides
+        info.suboffsets = NULL
+        info.itemsize = self.itemsize
+        info.format = self.format
+        # we do not need to call releasebuffer
+        info.obj = None
+
+    def __releasebuffer__(array self, Py_buffer* info):
+        # array.__releasebuffer__ should not be called,
+        # because the Py_buffer's 'obj' field is set to None.
+        raise NotImplementedError()
+
+    def __dealloc__(array self):
+        if self.data:
+            if self.callback_free_data != NULL:
+                self.callback_free_data(self.data)
+            else:
+                free(self.data)
+            self.data = NULL
+        if self.strides:
+            free(self.strides)
+            self.strides = NULL
+        if self.shape:
+            free(self.shape)
+            self.shape = NULL
+        self.format = NULL
+        self.itemsize = 0
+
+@cname("__pyx_array_new")
+cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode):
+    return array(shape, itemsize, format, mode.decode('ASCII'))
+
+# UtilityCode: MemoryView
+
+@cname('__pyx_MemviewEnum')
+cdef class Enum(object):
+    cdef object name
+    def __init__(self, name):
+        self.name = name
+    def __repr__(self):
+        return self.name
+
+cdef strided = Enum("<strided axis packing mode>")
+cdef contig = Enum("<contig axis packing mode>")
+cdef follow = Enum("<follow axis packing mode>")
+cdef direct = Enum("<direct axis access mode>")
+cdef ptr = Enum("<ptr axis access mode>")
+cdef full = Enum("<full axis access mode>")
+
+cdef extern from *:
+    int __Pyx_GetBuffer(object, Py_buffer *, int)
+    void __Pyx_ReleaseBuffer(Py_buffer *)
+
+@cname('__pyx_memoryview')
+cdef class memoryview(object):
+
+    cdef object obj
+    cdef Py_buffer view
+    cdef int gotbuf_flag
+
+    def __cinit__(memoryview self, object obj, int flags):
+        self.obj = obj
+        __Pyx_GetBuffer(self.obj, &self.view, flags)
+
+    def __dealloc__(memoryview self):
+        self.obj = None
+        __Pyx_ReleaseBuffer(&self.view)
+
+@cname('__pyx_memoryview_new')
+cdef memoryview memoryview_cwrapper(object o, int flags):
+    return memoryview(o, flags)

Cython/Utility/TestCythonScope.pyx

+# UtilityCode: TestClass
+# These utilities are for testing purposes
+
+cdef extern from *:
+    cdef object __pyx_test_dep(object)
+
+@cname('__pyx_TestClass')
+cdef class TestClass(object):
+    cdef public int value
+
+    def __init__(self, int value):
+        self.value = value
+
+    def __str__(self):
+        return 'TestClass(%d)' % self.value
+
+    cdef cdef_method(self, int value):
+        print 'Hello from cdef_method', value
+
+    cpdef cpdef_method(self, int value):
+        print 'Hello from cpdef_method', value
+
+    def def_method(self, int value):
+        print 'Hello from def_method', value
+
+    @cname('cdef_cname')
+    cdef cdef_cname_method(self, int value):
+        print "Hello from cdef_cname_method", value
+
+    @cname('cpdef_cname')
+    cpdef cpdef_cname_method(self, int value):
+        print "Hello from cpdef_cname_method", value
+
+    @cname('def_cname')
+    def def_cname_method(self, int value):
+        print "Hello from def_cname_method", value
+
+@cname('__pyx_test_call_other_cy_util')
+cdef test_call(obj):
+    print 'test_call'
+    __pyx_test_dep(obj)
+
+@cname('__pyx_TestClass_New')
+cdef _testclass_new(int value):
+    return TestClass(value)
+
+# UtilityCode: TestDep
+
+@cname('__pyx_test_dep')
+cdef test_dep(obj):
+    print 'test_dep', obj
+
+# UtilityCode: TestScope
+
+@cname('__pyx_testscope')
+cdef object _testscope(int value):
+    return "hello from cython scope, value=%d" % value
+
+# UtilityCode: View.TestScope
+
+@cname('__pyx_view_testscope')
+cdef object _testscope(int value):
+    return "hello from cython.view scope, value=%d" % value
+

MANIFEST.in

@@ -17,7 +17,8 @@ include Demos/embed/*
 include Demos/freeze/*
 include Demos/libraries/*
 include Demos/Makefile*
-recursive-include Cython/Debugger/Tests *
+recursive-include Cython/Debugger/Tests *.pyx *.pxd *.c *.h
+recursive-include Cython/Utility *
 recursive-include Tools *
 recursive-include tests *.pyx *.pxd *.pxi *.py *.h *.BROKEN bugs.txt
 recursive-include tests *_lib.cpp *.srctree

runtests.py

@@ -1187,6 +1187,7 @@ def refactor_for_py3(distdir, cy3_dir):
                      recursive-exclude Cython *
                      recursive-include Cython *.py *.pyx *.pxd
                      recursive-include Cython/Debugger/Tests *
+                     recursive-include Cython/Utility *
                      include runtests.py
                      include cython.py
                      ''')

setup.py

@@ -63,6 +63,7 @@ else:
         'Cython.Plex'     : ['*.pxd'],
         'Cython.Compiler' : ['*.pxd'],
         'Cython.Runtime'  : ['*.pyx', '*.pxd'],
+        'Cython.Utility'  : ['*.pyx', '*.pxd', '*.c', '*.h'],
         'Cython'          : [ p[7:] for p in pxd_include_patterns ],
         }
 
@@ -272,6 +273,7 @@ packages = [
     'Cython.Tests',
     'Cython.Build.Tests',
     'Cython.Compiler.Tests',
+    'Cython.Utility',
 ]
 
 if include_debugger:
@@ -280,7 +282,6 @@ if include_debugger:
     # it's enough to do this for Py2.5+:
     setup_args['package_data']['Cython.Debugger.Tests'] = ['codefile', 'cfuncs.c']
 
-
 setup(
   name = 'Cython',
   version = version,

tests/run/cythonarray.pyx

+# mode: run
+
+from __future__ import unicode_literals
+
 # from cython cimport array
 # cimport cython.array as array
 cimport cython as cy