Merge pull request #82 from markflorisson88/copy2

Atomic acquisition counting, slice assignment (+scalars), overlapping memory, broadcasting, more memoryview object methods, optimized with gil block

Cython/Compiler/Buffer.py

@@ -748,7 +748,7 @@ def get_type_information_cname(code, dtype, maxdepth=None):
         typeinfo = ('static __Pyx_TypeInfo %s = '
                         '{ "%s", %s, sizeof(%s), { %s }, %s, \'%s\', %s, %s };')
         tup = (name, rep, structinfo_name, declcode,
-               ', '.join([str(x) for x in arraysizes]), len(arraysizes),
+               ', '.join([str(x) for x in arraysizes]) or '0', len(arraysizes),
                typegroup, is_unsigned, flags)
         typecode.putln(typeinfo % tup, safe=True)
 

Cython/Compiler/Nodes.py

@@ -1460,7 +1460,8 @@ class FuncDefNode(StatNode, BlockNode):
             if self.return_type.is_pyobject:
                 init = " = NULL"
             elif self.return_type.is_memoryviewslice:
-                init = "= {0, 0}"
+                import MemoryView
+                init = ' = ' + MemoryView.memslice_entry_init
 
             code.putln(
                 "%s%s;" %
@@ -1479,20 +1480,47 @@ class FuncDefNode(StatNode, BlockNode):
         # ----- GIL acquisition
         acquire_gil = self.acquire_gil
 
-        # See if we need to acquire the GIL for variable declarations and
-        acquire_gil_for_var_decls_only = (lenv.nogil and
-                                          lenv.has_with_gil_block)
+        # See if we need to acquire the GIL for variable declarations, or for
+        # refnanny only
+
+        # Profiling or closures are not currently possible for cdef nogil
+        # functions, but check them anyway
+        have_object_args = (self.needs_closure or self.needs_outer_scope or
+                            profile)
+        for arg in lenv.arg_entries:
+            if arg.type.is_pyobject:
+                have_object_args = True
+                break
+
+        acquire_gil_for_var_decls_only = (
+                lenv.nogil and lenv.has_with_gil_block and
+                (have_object_args or lenv.buffer_entries))
 
-        use_refnanny = not lenv.nogil or acquire_gil_for_var_decls_only
+        acquire_gil_for_refnanny_only = (
+                lenv.nogil and lenv.has_with_gil_block and not
+                acquire_gil_for_var_decls_only)
+
+        use_refnanny = not lenv.nogil or lenv.has_with_gil_block
 
         if acquire_gil or acquire_gil_for_var_decls_only:
             code.put_ensure_gil()
 
         # ----- set up refnanny
         if use_refnanny:
+            if acquire_gil_for_refnanny_only:
+                code.declare_gilstate()
+                code.putln("#if CYTHON_REFNANNY")
+                code.put_ensure_gil(declare_gilstate=False)
+                code.putln("#endif /* CYTHON_REFNANNY */")
+
             tempvardecl_code.put_declare_refcount_context()
             code.put_setup_refcount_context(self.entry.name)
 
+            if acquire_gil_for_refnanny_only:
+                code.putln("#if CYTHON_REFNANNY")
+                code.put_release_ensured_gil()
+                code.putln("#endif /* CYTHON_REFNANNY */")
+
         # ----- Automatic lead-ins for certain special functions
         if is_getbuffer_slot:
             self.getbuffer_init(code)
@@ -1510,7 +1538,7 @@ class FuncDefNode(StatNode, BlockNode):
 
             if use_refnanny:
                 code.put_finish_refcount_context()
-                if acquire_gil_for_var_decls_only:
+                if acquire_gil or acquire_gil_for_var_decls_only:
                     code.put_release_ensured_gil()
 
             # FIXME: what if the error return value is a Python value?
@@ -1551,11 +1579,12 @@ class FuncDefNode(StatNode, BlockNode):
                     not entry.in_closure):
                     code.put_var_incref(entry)
 
-            # Note: defaults are always increffed. For def functions, we
+            # Note: defaults are always incref-ed. For def functions, we
             #       we aquire arguments from object converstion, so we have
             #       new references. If we are a cdef function, we need to
             #       incref our arguments
-            if is_cdef and entry.type.is_memoryviewslice:
+            elif (is_cdef and entry.type.is_memoryviewslice and
+                  len(entry.cf_assignments) > 1):
                 code.put_incref_memoryviewslice(entry.cname,
                                                 have_gil=not lenv.nogil)
         for entry in lenv.var_entries:
@@ -1566,10 +1595,6 @@ class FuncDefNode(StatNode, BlockNode):
         for entry in lenv.var_entries + lenv.arg_entries:
             if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
                 Buffer.put_init_vars(entry, code)
-        # ----- Initialise local memoryviewslices
-        for entry in lenv.var_entries:
-            if entry.visibility == "private" and not entry.used:
-                continue
 
         # ----- Check and convert arguments
         self.generate_argument_type_tests(code)
@@ -1631,13 +1656,13 @@ class FuncDefNode(StatNode, BlockNode):
                 # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
                 # code.put_trace_exception()
 
-                if lenv.nogil:
+                if lenv.nogil and not lenv.has_with_gil_block:
                     code.putln("{")
                     code.put_ensure_gil()
 
                 code.put_add_traceback(self.entry.qualified_name)
 
-                if lenv.nogil:
+                if lenv.nogil and not lenv.has_with_gil_block:
                     code.put_release_ensured_gil()
                     code.putln("}")
             else:
@@ -1715,7 +1740,10 @@ class FuncDefNode(StatNode, BlockNode):
                 if ((acquire_gil or len(entry.cf_assignments) > 1) and
                     not entry.in_closure):
                     code.put_var_decref(entry)
-            if entry.type.is_memoryviewslice:
+            elif (entry.type.is_memoryviewslice and
+                  (not is_cdef or len(entry.cf_assignments) > 1)):
+                # decref slices of def functions and acquired slices from cdef
+                # functions, but not borrowed slices from cdef functions.
                 code.put_xdecref_memoryviewslice(entry.cname,
                                                  have_gil=not lenv.nogil)
         if self.needs_closure:
@@ -1747,7 +1775,8 @@ class FuncDefNode(StatNode, BlockNode):
             # GIL holding funcion
             code.put_finish_refcount_context()
 
-        if acquire_gil or acquire_gil_for_var_decls_only:
+        if (acquire_gil or acquire_gil_for_var_decls_only or
+                                acquire_gil_for_refnanny_only):
             code.put_release_ensured_gil()
 
         if not self.return_type.is_void:
@@ -3152,7 +3181,7 @@ class DefNodeWrapper(FuncDefNode):
         if self.signature.has_dummy_arg:
             args.append(Naming.self_cname)
         for arg in self.args:
-            if arg.hdr_type:
+            if arg.hdr_type and not (arg.type.is_memoryviewslice or arg.type.is_struct):
                 args.append(arg.type.cast_code(arg.entry.cname))
             else:
                 args.append(arg.entry.cname)
@@ -4724,10 +4753,28 @@ class SingleAssignmentNode(AssignmentNode):
             self.lhs.analyse_target_declaration(env)
 
     def analyse_types(self, env, use_temp = 0):
+        import ExprNodes
+
         self.rhs.analyse_types(env)
         self.lhs.analyse_target_types(env)
         self.lhs.gil_assignment_check(env)
-        self.rhs = self.rhs.coerce_to(self.lhs.type, env)
+
+        if self.lhs.memslice_broadcast or self.rhs.memslice_broadcast:
+            self.lhs.memslice_broadcast = True
+            self.rhs.memslice_broadcast = True
+
+        is_index_node = isinstance(self.lhs, ExprNodes.IndexNode)
+        if (is_index_node and not self.rhs.type.is_memoryviewslice and
+            (self.lhs.memslice_slice or self.lhs.is_memslice_copy) and
+            (self.lhs.type.dtype.assignable_from(self.rhs.type) or
+             self.rhs.type.is_pyobject)):
+            # scalar slice assignment
+            self.lhs.is_memslice_scalar_assignment = True
+            dtype = self.lhs.type.dtype
+        else:
+            dtype = self.lhs.type
+
+        self.rhs = self.rhs.coerce_to(dtype, env)
         if use_temp:
             self.rhs = self.rhs.coerce_to_temp(env)
 
@@ -5168,7 +5215,6 @@ class ReturnStatNode(StatNode):
                         lhs_pos=self.value.pos,
                         rhs=self.value,
                         code=code,
-                        incref_rhs=self.value.is_name,
                         have_gil=self.in_nogil_context)
             else:
                 self.value.make_owned_reference(code)

Cython/Compiler/Options.py

@@ -67,9 +67,9 @@ old_style_globals = False
 cimport_from_pyx = False
 
 
-# max # of dims for buffers -- set to same value as max # of dims for numpy
-# arrays.
-buffer_max_dims = 32
+# max # of dims for buffers -- set lower than number of dimensions in numpy, as
+# slices are passed by value and involve a lot of copying
+buffer_max_dims = 8
 
 # Declare compiler directives
 directive_defaults = {

Cython/Compiler/ParseTreeTransforms.py

@@ -1778,6 +1778,9 @@ class AnalyseExpressionsTransform(CythonTransform):
 
         if node.is_fused_index and node.type is not PyrexTypes.error_type:
             node = node.base
+        elif node.memslice_ellipsis_noop:
+            # memoryviewslice[...] expression, drop the IndexNode
+            node = node.base
 
         return node
 

Cython/Compiler/PyrexTypes.py

@@ -510,7 +510,7 @@ class MemoryViewSliceType(PyrexType):
 
         return True
 
-    def declare_attribute(self, attribute, env):
+    def declare_attribute(self, attribute, env, pos):
         import MemoryView, Options
 
         scope = self.scope
@@ -518,24 +518,24 @@ class MemoryViewSliceType(PyrexType):
         if attribute == 'shape':
             scope.declare_var('shape',
                     c_array_type(c_py_ssize_t_type,
-                        Options.buffer_max_dims),
-                    None,
+                                 Options.buffer_max_dims),
+                    pos,
                     cname='shape',
                     is_cdef=1)
 
         elif attribute == 'strides':
             scope.declare_var('strides',
                     c_array_type(c_py_ssize_t_type,
-                        Options.buffer_max_dims),
-                    None,
+                                 Options.buffer_max_dims),
+                    pos,
                     cname='strides',
                     is_cdef=1)
 
         elif attribute == 'suboffsets':
             scope.declare_var('suboffsets',
                     c_array_type(c_py_ssize_t_type,
-                        Options.buffer_max_dims),
-                    None,
+                                 Options.buffer_max_dims),
+                    pos,
                     cname='suboffsets',
                     is_cdef=1)
 
@@ -544,40 +544,32 @@ class MemoryViewSliceType(PyrexType):
 
             to_axes_c = [('direct', 'contig')]
             to_axes_f = [('direct', 'contig')]
-            if ndim-1:
+            if ndim - 1:
                 to_axes_c = [('direct', 'follow')]*(ndim-1) + to_axes_c
                 to_axes_f = to_axes_f + [('direct', 'follow')]*(ndim-1)
 
             to_memview_c = MemoryViewSliceType(self.dtype, to_axes_c)
             to_memview_f = MemoryViewSliceType(self.dtype, to_axes_f)
 
-            cython_name_c, cython_name_f = "copy", "copy_fortran"
-            copy_name_c, copy_name_f = (
-                    MemoryView.get_copy_func_name(to_memview_c),
-                    MemoryView.get_copy_func_name(to_memview_f))
-
-
-            for (to_memview, cython_name, copy_name) in ((to_memview_c, cython_name_c, copy_name_c),
-                                                         (to_memview_f, cython_name_f, copy_name_f)):
-
+            for to_memview, cython_name in [(to_memview_c, "copy"),
+                                            (to_memview_f, "copy_fortran")]:
                 entry = scope.declare_cfunction(cython_name,
-                            CFuncType(self,
-                                [CFuncTypeArg("memviewslice", self, None)]),
-                            pos = None,
-                            defining = 1,
-                            cname = copy_name)
+                            CFuncType(self, [CFuncTypeArg("memviewslice", self, None)]),
+                            pos=pos,
+                            defining=1,
+                            cname=MemoryView.copy_c_or_fortran_cname(to_memview))
 
-                entry.utility_code_definition = \
-                        MemoryView.CopyFuncUtilCode(self, to_memview)
+                #entry.utility_code_definition = \
+                env.use_utility_code(MemoryView.get_copy_new_utility(pos, self, to_memview))
 
             MemoryView.use_cython_array_utility_code(env)
 
         elif attribute in ("is_c_contig", "is_f_contig"):
             # is_c_contig and is_f_contig functions
-            for (c_or_f, cython_name) in (('c', 'is_c_contig'), ('fortran', 'is_f_contig')):
+            for (c_or_f, cython_name) in (('c', 'is_c_contig'), ('f', 'is_f_contig')):
 
                 is_contig_name = \
-                        MemoryView.get_is_contig_func_name(c_or_f)
+                        MemoryView.get_is_contig_func_name(c_or_f, self.ndim)
 
                 cfunctype = CFuncType(
                         return_type=c_int_type,
@@ -587,14 +579,12 @@ class MemoryViewSliceType(PyrexType):
 
                 entry = scope.declare_cfunction(cython_name,
                             cfunctype,
-                            pos = None,
-                            defining = 1,
-                            cname = is_contig_name)
+                            pos=pos,
+                            defining=1,
+                            cname=is_contig_name)
 
-                if attribute == 'is_c_contig':
-                    entry.utility_code_definition = MemoryView.is_c_contig_utility
-                else:
-                    entry.utility_code_definition = MemoryView.is_f_contig_utility
+                entry.utility_code_definition = MemoryView.get_is_contig_utility(
+                                            attribute == 'is_c_contig', self.ndim)
 
         return True
 
@@ -604,10 +594,6 @@ class MemoryViewSliceType(PyrexType):
     def can_coerce_to_pyobject(self, env):
         return True
 
-    #def global_init_code(self, entry, code):
-    #    code.putln("%s.data = NULL;" % entry.cname)
-    #    code.putln("%s.memview = NULL;" % entry.cname)
-
     def check_for_null_code(self, cname):
         return cname + '.memview'
 
@@ -657,8 +643,9 @@ class MemoryViewSliceType(PyrexType):
         to_py_func = "(PyObject *(*)(char *)) " + to_py_func
         from_py_func = "(int (*)(char *, PyObject *)) " + from_py_func
 
-        tup = (obj.result(), self.ndim, to_py_func, from_py_func)
-        return "__pyx_memoryview_fromslice(&%s, %s, %s, %s);" % tup
+        tup = (obj.result(), self.ndim, to_py_func, from_py_func,
+               self.dtype.is_pyobject)
+        return "__pyx_memoryview_fromslice(&%s, %s, %s, %s, %d);" % tup
 
     def dtype_object_conversion_funcs(self, env):
         import MemoryView, Code

Cython/Utility/ModuleSetupCode.c

@@ -71,6 +71,8 @@
   #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)
   #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)
   #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)
+  #define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_FORMAT | PyBUF_WRITABLE)
+  #define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_FORMAT | PyBUF_WRITABLE)
 
 #endif
 

tests/run/memoryview.pyx

@@ -652,6 +652,11 @@ def test_indirect_slicing(arg):
     (5, 3, 2)
     0 0 -1
     58
+    56
+    58
+    58
+    58
+    58
     released A
 
     >>> test_indirect_slicing(IntMockBuffer("A", shape_9_14_21_list, shape=(9, 14, 21)))
@@ -659,6 +664,11 @@ def test_indirect_slicing(arg):
     (5, 14, 3)
     0 16 -1
     2412
+    2410
+    2412
+    2412
+    2412
+    2412
     released A
     """
     cdef int[::view.indirect, ::view.indirect, :] _a = arg
@@ -669,6 +679,11 @@ def test_indirect_slicing(arg):
     print_int_offsets(*b.suboffsets)
 
     print b[4, 2, 1]
+    print b[..., 0][4, 2]
+    print b[..., 1][4, 2]
+    print b[..., 1][4][2]
+    print b[4][2][1]
+    print b[4, 2][1]
 
 def test_direct_slicing(arg):
     """

tests/run/memoryviewattrs.pyx

@@ -74,6 +74,37 @@ def test_copy_to():
         print to_data[i],
     print
 
+@testcase
+def test_overlapping_copy():
+    """
+    >>> test_overlapping_copy()
+    """
+    cdef int i, array[10]
+    for i in range(10):
+        array[i] = i
+
+    cdef int[:] slice = array
+    slice[...] = slice[::-1]
+
+    for i in range(10):
+        assert slice[i] == 10 - 1 - i
+
+@testcase
+def test_partly_overlapping():
+    """
+    >>> test_partly_overlapping()
+    """
+    cdef int i, array[10]
+    for i in range(10):
+        array[i] = i
+
+    cdef int[:] slice = array
+    cdef int[:] slice2 = slice[:5]
+    slice2[...] = slice[4:9]
+
+    for i in range(5):
+        assert slice2[i] == i + 4
+
 @testcase
 @cython.nonecheck(True)
 def test_nonecheck1():
@@ -139,11 +170,11 @@ def test_copy_mismatch():
     u'''
     >>> test_copy_mismatch()
     Traceback (most recent call last):
-      ...
-    ValueError: memoryview shapes not the same in dimension 0
+       ...
+    ValueError: got differing extents in dimension 0 (got 2 and 3)
     '''
     cdef int[:,:,::1] mv1  = array((2,2,3), sizeof(int), 'i')
-    cdef int[:,:,::1] mv2  = array((1,2,3), sizeof(int), 'i')
+    cdef int[:,:,::1] mv2  = array((3,2,3), sizeof(int), 'i')
 
     mv1[...] = mv2
 

tests/run/numpy_memoryview.pyx

@@ -189,24 +189,46 @@ def test_transpose():
 
     print a[3, 2], a.T[2, 3], a_obj[3, 2], a_obj.T[2, 3], numpy_obj[3, 2], numpy_obj.T[2, 3]
 
-@testcase
+@testcase_numpy_1_5
 def test_numpy_like_attributes(cyarray):
     """
+    For some reason this fails in numpy 1.4, with shape () and strides (40, 8)
+    instead of 20, 4 on my machine. Investigate this.
+
     >>> cyarray = create_array(shape=(8, 5), mode="c")
     >>> test_numpy_like_attributes(cyarray)
     >>> test_numpy_like_attributes(cyarray.memview)
     """
     numarray = np.asarray(cyarray)
 
-    assert cyarray.shape == numarray.shape
-    assert cyarray.strides == numarray.strides
-    assert cyarray.ndim == numarray.ndim
-    assert cyarray.size == numarray.size
-    assert cyarray.nbytes == numarray.nbytes
+    assert cyarray.shape == numarray.shape, (cyarray.shape, numarray.shape)
+    assert cyarray.strides == numarray.strides, (cyarray.strides, numarray.strides)
+    assert cyarray.ndim == numarray.ndim, (cyarray.ndim, numarray.ndim)
+    assert cyarray.size == numarray.size, (cyarray.size, numarray.size)
+    assert cyarray.nbytes == numarray.nbytes, (cyarray.nbytes, numarray.nbytes)
 
     cdef int[:, :] mslice = numarray
     assert (<object> mslice).base is numarray
 
+@testcase_numpy_1_5
+def test_copy_and_contig_attributes(a):
+    """
+    >>> a = np.arange(20, dtype=np.int32).reshape(5, 4)
+    >>> test_copy_and_contig_attributes(a)
+    """
+    cdef np.int32_t[:, :] mslice = a
+    m = mslice
+
+    # Test object copy attributes
+    assert np.all(a == np.array(m.copy()))
+    assert a.strides == m.strides == m.copy().strides
+
+    assert np.all(a == np.array(m.copy_fortran()))
+    assert m.copy_fortran().strides == (4, 20)
+
+    # Test object is_*_contig attributes
+    assert m.is_c_contig() and m.copy().is_c_contig()
+    assert m.copy_fortran().is_f_contig() and not m.is_f_contig()
 
 ctypedef int td_cy_int
 cdef extern from "bufaccess.h":