cimport cython in CyUtility, more tests, nogil slicing

Cython/Compiler/Code.pxd

@@ -13,6 +13,8 @@ cdef class UtilityCode(UtilityCodeBase):
     cdef public dict _cache
     cdef public list specialize_list
     cdef public object proto_block
+    cdef public object name
+    cdef public object file
 
     cpdef put_code(self, output)
 

Cython/Compiler/CythonScope.py

@@ -75,10 +75,13 @@ class CythonScope(ModuleScope):
         Creates some entries for testing purposes and entries for
         cython.array() and for cython.view.*.
         """
-        cython_testscope_utility_code.declare_in_scope(self)
-        cython_test_extclass_utility_code.declare_in_scope(self)
+        cython_testscope_utility_code.declare_in_scope(
+                                self, cython_scope=self)
+        cython_test_extclass_utility_code.declare_in_scope(
+                                    self, cython_scope=self)
 
-        MemoryView.cython_array_utility_code.declare_in_scope(self)
+        MemoryView.cython_array_utility_code.declare_in_scope(
+                                        self, cython_scope=self)
 
         #
         # The view sub-scope
@@ -88,9 +91,11 @@ class CythonScope(ModuleScope):
         viewscope.is_cython_builtin = True
         viewscope.pxd_file_loaded = True
 
-        cythonview_testscope_utility_code.declare_in_scope(viewscope)
+        cythonview_testscope_utility_code.declare_in_scope(
+                                                viewscope, cython_scope=self)
 
-        view_utility_scope = MemoryView.view_utility_code.declare_in_scope(viewscope)
+        view_utility_scope = MemoryView.view_utility_code.declare_in_scope(
+                                                viewscope, cython_scope=self)
         # MemoryView.memview_fromslice_utility_code.from_scope = view_utility_scope
         # MemoryView.memview_fromslice_utility_code.declare_in_scope(viewscope)
 

Cython/Compiler/ExprNodes.py

@@ -2432,7 +2432,8 @@ class IndexNode(ExprNode):
             import MemoryView
 
             skip_child_analysis = True
-            indices = MemoryView.unellipsify(indices, self.base.type.ndim)
+            have_slices, indices = MemoryView.unellipsify(indices,
+                                                          self.base.type.ndim)
             self.memslice_index = len(indices) == self.base.type.ndim
             axes = []
 
@@ -2462,24 +2463,19 @@ class IndexNode(ExprNode):
                         value = getattr(index, attr)
                         if not value.is_none:
                             value = value.coerce_to(index_type, env)
-                            value = value.coerce_to_temp(env)
+                            #value = value.coerce_to_temp(env)
                             setattr(index, attr, value)
                             new_indices.append(value)
 
                 elif index.type.is_int:
                     self.memslice_index = True
-                    index = index.coerce_to(index_type, env).coerce_to_temp(
-                                                                 index_type)
+                    index = index.coerce_to(index_type, env)\
+                    #.coerce_to_temp(
+                    #                                             index_type)
                     indices[i] = index
                     new_indices.append(index)
 
-                    if access in ('ptr', 'generic') and i != 0:
-                        # If this dimension is to disappear, then how do we
-                        # indicate that we need to dereference in this dimension
-                        # if the previous dimension is already indirect, or if
-                        # the previous dimension was direct but also indexed?
-                        # Basically only a[i, j, k, :] can work, as you can
-                        # set the base pointer to start in the fourth dimension
+                    if access in ('ptr', 'generic') and i != 0 and have_slices:
                         self.type = error_type
                         return error(index.pos,
                                      "Indexing of non-leading indirect or generic "
@@ -2494,6 +2490,8 @@ class IndexNode(ExprNode):
             self.original_indices = indices
             self.indices = new_indices
 
+            self.env = env
+
         elif self.base.type.is_buffer:
             # Buffer indexing
             if len(indices) == self.base.type.ndim:
@@ -2893,7 +2891,8 @@ class IndexNode(ExprNode):
                                                 self.original_indices,
                                                 self.base.type,
                                                 self.type,
-                                                self.result())
+                                                self.result(),
+                                                have_gil = not self.env.nogil)
 
     def put_nonecheck(self, code):
         code.globalstate.use_utility_code(raise_noneindex_error_utility_code)
@@ -4092,6 +4091,7 @@ class AttributeNode(ExprNode):
     def nogil_check(self, env):
         if self.is_py_attr:
             self.gil_error()
+        elif self.type.is_memoryviewslice:
             import MemoryView
             MemoryView.err_if_nogil_initialized_check(self.pos, env, 'attribute')
 

Cython/Compiler/UtilityCode.py

@@ -9,6 +9,7 @@ class NonManglingModuleScope(Symtab.ModuleScope):
 
     def __init__(self, prefix, *args, **kw):
         self.prefix = prefix
+        self.cython_scope = None
         Symtab.ModuleScope.__init__(self, *args, **kw)
 
     def add_imported_entry(self, name, entry, pos):
@@ -31,9 +32,12 @@ class CythonUtilityCodeContext(StringParseContext):
 
     def find_module(self, module_name, relative_to = None, pos = None,
                     need_pxd = 1):
+
         if module_name != self.module_name:
-            raise AssertionError("Not yet supporting any cimports/includes "
-                                 "from string code snippets")
+            if module_name not in self.modules:
+                raise AssertionError("Only the cython cimport is supported.")
+            else:
+                return self.modules[module_name]
 
         if self.scope is None:
             self.scope = NonManglingModuleScope(self.prefix,
@@ -78,7 +82,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
         self.requires = requires or []
         self.from_scope = from_scope
 
-    def get_tree(self, entries_only=False):
+    def get_tree(self, entries_only=False, cython_scope=None):
         from AnalysedTreeTransforms import AutoTestDictTransform
         # The AutoTestDictTransform creates the statement "__test__ = {}",
         # which when copied into the main ModuleNode overwrites
@@ -88,6 +92,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
         import Pipeline, ParseTreeTransforms
         context = CythonUtilityCodeContext(self.name)
         context.prefix = self.prefix
+        context.cython_scope = cython_scope
         #context = StringParseContext(self.name)
         tree = parse_from_strings(self.name, self.impl, context=context,
                                   allow_struct_enum_decorator=True)
@@ -125,13 +130,13 @@ class CythonUtilityCode(Code.UtilityCodeBase):
     def put_code(self, output):
         pass
 
-    def declare_in_scope(self, dest_scope, used=False):
+    def declare_in_scope(self, dest_scope, used=False, cython_scope=None):
         """
         Declare all entries from the utility code in dest_scope. Code will only
         be included for used entries. If module_name is given, declare the
         type entries with that name.
         """
-        tree = self.get_tree(entries_only=True)
+        tree = self.get_tree(entries_only=True, cython_scope=cython_scope)
 
         entries = tree.scope.entries
         entries.pop('__name__')

Cython/Utility/MemoryView_C.c

@@ -284,7 +284,7 @@ static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
     if (!memview)
         return; /* allow uninitialized memoryview assignment */
 
-    if (memview->acquisition_count <= 0)
+    if (memview->acquisition_count < 0)
         __pyx_fatalerror("Acquisition count is %d (line %d)",
                          memview->acquisition_count, lineno);
 

tests/run/memoryview.pyx

@@ -588,40 +588,122 @@ def assign_temporary_to_object(object[:] mslice):
     buf = mslice
     buf[1] = {3-2: 2+(2*4)-2}
 
-def test_slicing(arg):
+def print_int_offsets(*args):
+    for item in args:
+        print item / sizeof(int),
+
+    print
+
+def test_generic_slicing(arg):
     """
     Test simple slicing
-    >>> test_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
+    >>> test_generic_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
     acquired A
-    3 9 2
-    1232 -44 4
+    (3, 9, 2)
+    308 -11 1
     -1 -1 -1
     released A
 
     Test direct slicing, negative slice oob in dim 2
-    >>> test_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
+    >>> test_generic_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
     acquired A
-    0 0 2
-    48 -12 4
+    (0, 0, 2)
+    12 -3 1
     -1 -1 -1
     released A
 
     Test indirect slicing
     >>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
-    >>> test_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
+    >>> test_generic_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
     acquired A
-    2 0 2
-    8 -4 4
-    0 0 -1
+    (2, 0, 2)
+    0 1 -1
     released A
+
+    >>> stride1 = 21 * 14
+    >>> stride2 = 21
+    >>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
+    >>> test_generic_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
+    acquired A
+    (3, 9, 2)
+    20 1 -1
+    released A
+
     """
     cdef int[::view.generic, ::view.generic, :] _a = arg
     a = _a
     b = a[2:8:2, -4:1:-1, 1:3]
 
-    print b.shape[0], b.shape[1], b.shape[2]
-    print b.strides[0], b.strides[1], b.strides[2]
-    print b.suboffsets[0], b.suboffsets[1], b.suboffsets[2]
+    print b.shape
+    if b.suboffsets[0] < 0:
+        print_int_offsets(*b.strides)
+    print_int_offsets(*b.suboffsets)
+
+    cdef int i, j, k
+    for i in range(b.shape[0]):
+        for j in range(b.shape[1]):
+            for k in range(b.shape[2]):
+                itemA = a[2 + 2 * i, -4 - j, 1 + k]
+                itemB = b[i, j, k]
+                assert itemA == itemB, (i, j, k, itemA, itemB)
+
+def test_indirect_slicing(arg):
+    """
+    Test indirect slicing
+    >>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
+    >>> test_indirect_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
+    acquired A
+    (5, 3, 2)
+    0 0 -1
+    58
+    released A
+
+    >>> stride1 = 21 * 14
+    >>> stride2 = 21
+    >>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
+    >>> test_indirect_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
+    acquired A
+    (5, 14, 3)
+    0 16 -1
+    2412
+    released A
+    """
+    cdef int[::view.indirect, ::view.indirect, :] _a = arg
+    a = _a
+    b = a[-5:, ..., -5:100:2]
+
+    print b.shape
+    print_int_offsets(*b.suboffsets)
+
+    print b[4, 2, 1]
+
+def test_direct_slicing(arg):
+    """
+    Fused types would be convenient to test this stuff!
+
+    Test simple slicing
+    >>> test_direct_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
+    acquired A
+    (3, 9, 2)
+    308 -11 1
+    -1 -1 -1
+    released A
+
+    Test direct slicing, negative slice oob in dim 2
+    >>> test_direct_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
+    acquired A
+    (0, 0, 2)
+    12 -3 1
+    -1 -1 -1
+    released A
+    """
+    cdef int[:, :, :] _a = arg
+    a = _a
+    b = a[2:8:2, -4:1:-1, 1:3]
+
+    print b.shape
+    print_int_offsets(*b.strides)
+    print_int_offsets(*b.suboffsets)
 
     cdef int i, j, k
     for i in range(b.shape[0]):
@@ -631,13 +713,14 @@ def test_slicing(arg):
                 itemB = b[i, j, k]
                 assert itemA == itemB, (i, j, k, itemA, itemB)
 
+
 def test_slicing_and_indexing(arg):
     """
     >>> a = IntStridedMockBuffer("A", range(10 * 3 * 5), shape=(10, 3, 5))
     >>> test_slicing_and_indexing(a)
     acquired A
-    5 2
-    60 8
+    (5, 2)
+    15 2
     126 113
     [111]
     released A
@@ -648,8 +731,8 @@ def test_slicing_and_indexing(arg):
     c = b[4:1:-1, ::-1]
     d = c[2, 1:2]
 
-    print b.shape[0], b.shape[1]
-    print b.strides[0], b.strides[1]
+    print b.shape
+    print_int_offsets(*b.strides)
 
     cdef int i, j
     for i in range(b.shape[0]):
@@ -660,3 +743,13 @@ def test_slicing_and_indexing(arg):
 
     print c[1, 1], c[2, 0]
     print [d[i] for i in range(d.shape[0])]
+
+def test_oob():
+    """
+    >>> test_oob()
+    Traceback (most recent call last):
+       ...
+    IndexError: Index out of bounds (axis 1)
+    """
+    cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9))
+    print a[:, 20]

tests/run/memslice.pyx

@@ -2,6 +2,7 @@
 
 from __future__ import unicode_literals
 
+cimport cython
 from cython cimport view
 
 __test__ = {}
@@ -1158,40 +1159,127 @@ def test_cdef_function2():
 
     cdef_function2(global_A, global_B)
 
+
+def print_int_offsets(*args):
+    for item in args:
+        print item / sizeof(int),
+
+    print
+
 @testcase
-def test_slicing(arg):
+def test_generic_slicing(arg):
     """
     Test simple slicing
-    >>> test_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
+    >>> test_generic_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
     acquired A
     3 9 2
-    1232 -44 4
+    308 -11 1
     -1 -1 -1
     released A
 
     Test direct slicing, negative slice oob in dim 2
-    >>> test_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
+    >>> test_generic_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
     acquired A
     0 0 2
-    48 -12 4
+    12 -3 1
     -1 -1 -1
     released A
 
     Test indirect slicing
     >>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
-    >>> test_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
+    >>> test_generic_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
     acquired A
     2 0 2
-    8 -4 4
-    0 0 -1
+    0 1 -1
+    released A
+
+    >>> stride1 = 21 * 14
+    >>> stride2 = 21
+    >>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
+    >>> test_generic_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
+    acquired A
+    3 9 2
+    20 1 -1
     released A
+
     """
     cdef int[::view.generic, ::view.generic, :] a = arg
     cdef int[::view.generic, ::view.generic, :] b = a[2:8:2, -4:1:-1, 1:3]
 
     print b.shape[0], b.shape[1], b.shape[2]
-    print b.strides[0], b.strides[1], b.strides[2]
-    print b.suboffsets[0], b.suboffsets[1], b.suboffsets[2]
+    if b.suboffsets[0] < 0:
+        print_int_offsets(b.strides[0], b.strides[1], b.strides[2])
+    print_int_offsets(b.suboffsets[0], b.suboffsets[1], b.suboffsets[2])
+
+    cdef int i, j, k
+    for i in range(b.shape[0]):
+        for j in range(b.shape[1]):
+            for k in range(b.shape[2]):
+                itemA = a[2 + 2 * i, -4 - j, 1 + k]
+                itemB = b[i, j, k]
+                assert itemA == itemB, (i, j, k, itemA, itemB)
+
+@testcase
+def test_indirect_slicing(arg):
+    """
+    Test indirect slicing
+    >>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
+    >>> test_indirect_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
+    acquired A
+    5 3 2
+    0 0 -1
+    58
+    56
+    released A
+
+    >>> stride1 = 21 * 14
+    >>> stride2 = 21
+    >>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
+    >>> test_indirect_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
+    acquired A
+    5 14 3
+    0 16 -1
+    2412
+    2410
+    released A
+    """
+    cdef int[::view.indirect, ::view.indirect, :] a = arg
+    cdef int[::view.indirect, ::view.indirect, :] b = a[-5:, ..., -5:100:2]
+    cdef int[::view.indirect, ::view.indirect] c = b[..., 0]
+
+    print b.shape[0], b.shape[1], b.shape[2]
+    print_int_offsets(b.suboffsets[0], b.suboffsets[1], b.suboffsets[2])
+
+    print b[4, 2, 1]
+    print c[4, 2]
+
+@testcase
+def test_direct_slicing(arg):
+    """
+    Fused types would be convenient to test this stuff!
+
+    Test simple slicing
+    >>> test_direct_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
+    acquired A
+    3 9 2
+    308 -11 1
+    -1 -1 -1
+    released A
+
+    Test direct slicing, negative slice oob in dim 2
+    >>> test_direct_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
+    acquired A
+    0 0 2
+    12 -3 1
+    -1 -1 -1
+    released A
+    """
+    cdef int[:, :, ::1] a = arg
+    cdef int[:, :, :] b = a[2:8:2, -4:1:-1, 1:3]
+
+    print b.shape[0], b.shape[1], b.shape[2]
+    print_int_offsets(b.strides[0], b.strides[1], b.strides[2])
+    print_int_offsets(b.suboffsets[0], b.suboffsets[1], b.suboffsets[2])
 
     cdef int i, j, k
     for i in range(b.shape[0]):
@@ -1208,7 +1296,7 @@ def test_slicing_and_indexing(arg):
     >>> test_slicing_and_indexing(a)
     acquired A
     5 2
-    60 8
+    15 2
     126 113
     [111]
     released A
@@ -1219,7 +1307,7 @@ def test_slicing_and_indexing(arg):
     cdef int[:] d = c[2, 1:2]
 
     print b.shape[0], b.shape[1]
-    print b.strides[0], b.strides[1]
+    print_int_offsets(b.strides[0], b.strides[1])
 
     cdef int i, j
     for i in range(b.shape[0]):
@@ -1230,3 +1318,15 @@ def test_slicing_and_indexing(arg):
 
     print c[1, 1], c[2, 0]
     print [d[i] for i in range(d.shape[0])]
+
+
+@testcase
+def test_oob():
+    """
+    >>> test_oob()
+    Traceback (most recent call last):
+       ...
+    IndexError: Index out of bounds (axis 1)
+    """
+    cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9))
+    print a[:, 20]

tests/run/numpy_memoryview.pyx

@@ -6,13 +6,15 @@ Test slicing for memoryviews and memoryviewslices
 """
 
 cimport numpy as np
-import numpy
+import numpy as np
+
+ctypedef np.int32_t dtype_t
 
 def get_array():
     # We need to type our array to get a __pyx_get_buffer() that typechecks
     # for np.ndarray and calls __getbuffer__ in numpy.pxd
-    cdef np.ndarray[int, ndim=3] a
-    a = numpy.arange(8 * 14 * 11).reshape(8, 14, 11)
+    cdef np.ndarray[dtype_t, ndim=3] a
+    a = np.arange(8 * 14 * 11, dtype=np.int32).reshape(8, 14, 11)
     return a
 
 a = get_array()
@@ -31,11 +33,11 @@ def test_partial_slicing(array):
     """
     >>> test_partial_slicing(a)
     """
-    cdef int[:, :, :] a = array
+    cdef dtype_t[:, :, :] a = array
     obj = array[4]
 
-    cdef int[:, :] b = a[4, :]
-    cdef int[:, :] c = a[4]
+    cdef dtype_t[:, :] b = a[4, :]
+    cdef dtype_t[:, :] c = a[4]
 
     ae(b.shape[0], c.shape[0], obj.shape[0])
     ae(b.shape[1], c.shape[1], obj.shape[1])
@@ -46,15 +48,15 @@ def test_ellipsis(array):
     """
     >>> test_ellipsis(a)
     """
-    cdef int[:, :, :] a = array
+    cdef dtype_t[:, :, :] a = array
 
-    cdef int[:, :] b = a[..., 4]
+    cdef dtype_t[:, :] b = a[..., 4]
     b_obj = array[..., 4]
 
-    cdef int[:, :] c = a[4, ...]
+    cdef dtype_t[:, :] c = a[4, ...]
     c_obj = array[4, ...]
 
-    cdef int[:, :] d = a[2:8, ..., 2]
+    cdef dtype_t[:, :] d = a[2:8, ..., 2]
     d_obj = array[2:8, ..., 2]
 
     ae(tuple([b.shape[i] for i in range(2)]), b_obj.shape)
@@ -75,7 +77,7 @@ def test_ellipsis(array):
         for j in range(d.shape[1]):
             ae(d[i, j], d_obj[i, j])
 
-    cdef int[:] e = a[..., 5, 6]
+    cdef dtype_t[:] e = a[..., 5, 6]
     e_obj = array[..., 5, 6]
     ae(e.shape[0], e_obj.shape[0])
     ae(e.strides[0], e_obj.strides[0])
@@ -88,7 +90,7 @@ def test_partial_slicing_memoryview(array):
     """
     >>> test_partial_slicing_memoryview(a)
     """
-    cdef int[:, :, :] _a = array
+    cdef dtype_t[:, :, :] _a = array
     a = _a
     obj = array[4]
 
@@ -104,7 +106,7 @@ def test_ellipsis_memoryview(array):
     """
     >>> test_ellipsis_memoryview(a)
     """
-    cdef int[:, :, :] _a = array
+    cdef dtype_t[:, :, :] _a = array
     a = _a
 
     b = a[..., 4]