Support slicing memoryview slices

Cython/Compiler/Pipeline.py

@@ -192,7 +192,6 @@ def create_pipeline(context, mode, exclude_classes=()):
         FinalOptimizePhase(context),
         GilCheck(),
         UseUtilityCodeDefinitions(context),
-#        PrintTree(),
         ]
     filtered_stages = []
     for s in stages:
@@ -293,7 +292,9 @@ def insert_into_pipeline(pipeline, transform, before=None, after=None):
 # Running a pipeline
 #
 
-def run_pipeline(pipeline, source):
+def run_pipeline(pipeline, source, printtree=True):
+    from Cython.Compiler.Visitor import PrintTree
+
     error = None
     data = source
     try:
@@ -303,6 +304,8 @@ def run_pipeline(pipeline, source):
                     if DebugFlags.debug_verbose_pipeline:
                         t = time()
                         print "Entering pipeline phase %r" % phase
+                    if not printtree and isinstance(phase, PrintTree):
+                        continue
                     data = phase(data)
                     if DebugFlags.debug_verbose_pipeline:
                         print "    %.3f seconds" % (time() - t)

Cython/Compiler/PyrexTypes.py

@@ -187,7 +187,7 @@ class PyrexType(BaseType):
     def needs_nonecheck(self):
         return 0
 
-        
+
 def public_decl(base_code, dll_linkage):
     if dll_linkage:
         return "%s(%s)" % (dll_linkage, base_code)
@@ -414,8 +414,7 @@ class MemoryViewSliceType(PyrexType):
 
     def attributes_known(self):
         if self.scope is None:
-
-            import Symtab, MemoryView, Options
+            import Symtab
 
             self.scope = scope = Symtab.CClassScope(
                     'mvs_class_'+self.specialization_suffix(),
@@ -424,8 +423,17 @@ class MemoryViewSliceType(PyrexType):
 
             scope.parent_type = self
 
-            scope.declare_var('_data', c_char_ptr_type, None, cname='data', is_cdef=1)
+            scope.declare_var('_data', c_char_ptr_type, None,
+                              cname='data', is_cdef=1)
+
+        return True
+
+    def declare_attribute(self, attribute):
+        import MemoryView, Options
+
+        scope = self.scope
 
+        if attribute == 'shape':
             scope.declare_var('shape',
                     c_array_type(c_py_ssize_t_type,
                         Options.buffer_max_dims),
@@ -433,6 +441,7 @@ class MemoryViewSliceType(PyrexType):
                     cname='shape',
                     is_cdef=1)
 
+        elif attribute == 'strides':
             scope.declare_var('strides',
                     c_array_type(c_py_ssize_t_type,
                         Options.buffer_max_dims),
@@ -440,6 +449,7 @@ class MemoryViewSliceType(PyrexType):
                     cname='strides',
                     is_cdef=1)
 
+        elif attribute == 'suboffsets':
             scope.declare_var('suboffsets',
                     c_array_type(c_py_ssize_t_type,
                         Options.buffer_max_dims),
@@ -447,6 +457,7 @@ class MemoryViewSliceType(PyrexType):
                     cname='suboffsets',
                     is_cdef=1)
 
+        elif attribute in ("copy", "copy_fortran"):
             ndim = len(self.axes)
 
             to_axes_c = [('direct', 'contig')]

Cython/Compiler/UtilityCode.py

@@ -118,7 +118,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
             pipeline = Pipeline.insert_into_pipeline(pipeline, scope_transform,
                                                      before=transform)
 
-        (err, tree) = Pipeline.run_pipeline(pipeline, tree)
+        (err, tree) = Pipeline.run_pipeline(pipeline, tree, printtree=False)
         assert not err, err
         return tree
 

Cython/Utility/MemoryView_C.c

@@ -5,7 +5,6 @@
 
 typedef struct {
   struct {{memview_struct_name}} *memview;
-  /* For convenience and faster access */
   char *data;
   Py_ssize_t shape[{{max_dims}}];
   Py_ssize_t strides[{{max_dims}}];
@@ -30,9 +29,6 @@ static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *);
 #define __Pyx_IS_C_CONTIG 1
 #define __Pyx_IS_F_CONTIG 2
 
-
-/* #define __PYX_MEMSLICE_GETDATA(SLICE) ((char *) SLICE->memview->view->buf) */
-
 static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview,
                                 int *axes_specs, int c_or_f_flag,  int ndim, __Pyx_TypeInfo *dtype,
                                 __Pyx_BufFmt_StackElem stack[], __Pyx_memviewslice *memviewslice);
@@ -170,7 +166,7 @@ static int __Pyx_ValidateAndInit_memviewslice(
         }
 
         if (spec & __Pyx_MEMVIEW_PTR) {
-            if (buf->suboffsets && buf->suboffsets[i] < 0) {
+            if (!buf->suboffsets || (buf->suboffsets && buf->suboffsets[i] < 0)) {
                 PyErr_Format(PyExc_ValueError,
                     "Buffer is not indirectly accessisble in dimension %d.", i);
                 goto fail;
@@ -287,7 +283,6 @@ static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
     struct {{memview_struct_name}} *memview = memslice->memview;
     if (!memview)
         return; /* allow uninitialized memoryview assignment */
-        /* __pyx_fatalerror("memoryslice is not initialized (line %d)", lineno); */
 
     if (memview->acquisition_count <= 0)
         __pyx_fatalerror("Acquisition count is %d (line %d)",
@@ -324,6 +319,7 @@ static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
     last_time = (memview->acquisition_count-- == 1);
     PyThread_release_lock(memview->lock);
 
+    memslice->data = NULL;
     if (last_time) {
         if (have_gil) {
             Py_CLEAR(memslice->memview);
@@ -332,6 +328,7 @@ static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
             Py_CLEAR(memslice->memview);
             PyGILState_Release(_gilstate);
         }
+    } else {
         memslice->memview = NULL;
     }
 }
@@ -466,4 +463,4 @@ int {{set_function}}(const char *itemp, PyObject *obj) {
     Py_DECREF(*(PyObject **) itemp);
     *(PyObject **) itemp = obj;
     return 1;
-}
+}
\ No newline at end of file

tests/run/cythonarray.pyx

@@ -5,7 +5,8 @@ from __future__ import unicode_literals
 from cython cimport array
 cimport cython as cy
 
-from libc.stdlib cimport malloc, free
+include "cythonarrayutil.pxi"
+
 
 def contiguity():
     '''
@@ -67,26 +68,6 @@ def dont_allocate_buffer():
     result.callback_free_data = callback
     result = None
 
-cdef void callback(char *data):
-    print "callback called %d" % <long> data
-
-cdef create_array(shape, mode):
-    cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode)
-    cdef int *data = <int *> result.data
-    cdef int i, j, cidx, fidx
-
-    for i in range(shape[0]):
-        for j in range(shape[1]):
-            cidx = i * shape[1] + j
-            fidx = i + j * shape[0]
-
-            if mode == 'fortran':
-                data[fidx] = cidx
-            else:
-                data[cidx] = cidx
-
-    return result
-
 def test_cython_array_getbuffer():
     """
     >>> test_cython_array_getbuffer()

tests/run/cythonarrayutil.pxi

+from libc.stdlib cimport malloc, free
+cimport cython
+
+cdef void callback(char *data):
+    print "callback called %d" % <long> data
+
+def create_array(shape, mode, use_callback=False):
+    cdef cython.array result = cython.array(shape, itemsize=sizeof(int),
+                                            format='i', mode=mode)
+    cdef int *data = <int *> result.data
+    cdef int i, j, cidx, fidx
+
+    for i in range(shape[0]):
+        for j in range(shape[1]):
+            cidx = i * shape[1] + j
+            fidx = i + j * shape[0]
+
+            if mode == 'fortran':
+                data[fidx] = cidx
+            else:
+                data[cidx] = cidx
+
+    if use_callback:
+        result.callback_free_data = callback
+
+    return result

tests/run/memslice.pyx

@@ -23,6 +23,7 @@ def testcase(func):
 
 
 include "mockbuffers.pxi"
+include "cythonarrayutil.pxi"
 
 #
 # Buffer acquire and release tests
@@ -1083,12 +1084,14 @@ def buffer_nogil():
 #
 ### Test cdef functions
 #
-cdef cdef_function(int[:] buf1, object[::view.indirect, :] buf2 = ObjectMockBuffer(None,
-                                                            [["spam"],["ham"],["eggs"]])):
+
+objs = [["spam"], ["ham"], ["eggs"]]
+cdef cdef_function(int[:] buf1, object[::view.indirect, :] buf2 = ObjectMockBuffer(None, objs)):
     print 'cdef called'
     print buf1[6], buf2[1, 0]
     buf2[1, 0] = "eggs"
 
+@testcase
 def test_cdef_function(o1, o2=None):
     """
     >>> A = IntMockBuffer("A", range(10))
@@ -1101,19 +1104,22 @@ def test_cdef_function(o1, o2=None):
     cdef called
     6 eggs
     released A
-    >>> B = ObjectMockBuffer("B", range(25), shape=(5, 5))
+    >>> L = [[x] for x in range(25)]
+    >>> B = ObjectMockBuffer("B", L, shape=(5, 5))
     >>> test_cdef_function(A, B)
     acquired A
-    acquired B
     cdef called
     6 eggs
     released A
-    released B
     acquired A
-    acquired B
     cdef called
     6 eggs
     released A
+    acquired A
+    acquired B
+    cdef called
+    6 1
+    released A
     released B
     """
     cdef_function(o1)
@@ -1122,15 +1128,15 @@ def test_cdef_function(o1, o2=None):
     if o2:
         cdef_function(o1, o2)
 
-cdef int[:] global_A = IntMockBuffer("A", range(10))
-cdef object[::view.indirect, :] global_B = ObjectMockBuffer(
-                            None, [["spam"],["ham"],["eggs"]])
+cdef int[:] global_A = IntMockBuffer("Global_A", range(10))
+cdef object[::view.indirect, :] global_B = ObjectMockBuffer(None, objs)
 
 cdef cdef_function2(int[:] buf1, object[::view.indirect, :] buf2 = global_B):
     print 'cdef2 called'
     print buf1[6], buf2[1, 0]
     buf2[1, 0] = "eggs"
 
+@testcase
 def test_cdef_function2():
     """
     >>> test_cdef_function2()
@@ -1151,3 +1157,76 @@ def test_cdef_function2():
     print global_B[1, 0]
 
     cdef_function2(global_A, global_B)
+
+@testcase
+def test_slicing(arg):
+    """
+    Test simple slicing
+    >>> test_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
+    acquired A
+    3 9 2
+    1232 -44 4
+    -1 -1 -1
+    released A
+
+    Test direct slicing, negative slice oob in dim 2
+    >>> test_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
+    acquired A
+    0 0 2
+    48 -12 4
+    -1 -1 -1
+    released A
+
+    Test indirect slicing
+    >>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
+    >>> test_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
+    acquired A
+    2 0 2
+    8 -4 4
+    0 0 -1
+    released A
+    """
+    cdef int[::view.generic, ::view.generic, :] a = arg
+    cdef int[::view.generic, ::view.generic, :] b = a[2:8:2, -4:1:-1, 1:3]
+
+    print b.shape[0], b.shape[1], b.shape[2]
+    print b.strides[0], b.strides[1], b.strides[2]
+    print b.suboffsets[0], b.suboffsets[1], b.suboffsets[2]
+
+    cdef int i, j, k
+    for i in range(b.shape[0]):
+        for j in range(b.shape[1]):
+            for k in range(b.shape[2]):
+                itemA = a[2 + 2 * i, -4 - j, 1 + k]
+                itemB = b[i, j, k]
+                assert itemA == itemB, (i, j, k, itemA, itemB)
+
+@testcase
+def test_slicing_and_indexing(arg):
+    """
+    >>> a = IntStridedMockBuffer("A", range(10 * 3 * 5), shape=(10, 3, 5))
+    >>> test_slicing_and_indexing(a)
+    acquired A
+    5 2
+    60 8
+    126 113
+    [111]
+    released A
+    """
+    cdef int[:, :, :] a = arg
+    cdef int[:, :] b = a[-5:, 1, 1::2]
+    cdef int[:, :] c = b[4:1:-1, ::-1]
+    cdef int[:] d = c[2, 1:2]
+
+    print b.shape[0], b.shape[1]
+    print b.strides[0], b.strides[1]
+
+    cdef int i, j
+    for i in range(b.shape[0]):
+        for j in range(b.shape[1]):
+            itemA = a[-5 + i, 1, 1 + 2 * j]
+            itemB = b[i, j]
+            assert itemA == itemB, (i, j, itemA, itemB)
+
+    print c[1, 1], c[2, 0]
+    print [d[i] for i in range(d.shape[0])]
\ No newline at end of file