Memoryview slice item assignment, better errors, pass all tests

Cython/Compiler/ExprNodes.py

@@ -594,8 +594,14 @@ class ExprNode(Node):
                           "Cannot convert '%s' to memoryviewslice" %
                                                                 (src_type,))
             elif not MemoryView.src_conforms_to_dst(src.type, dst_type):
-                error(self.pos, "Memoryview '%s' not conformable to memoryview '%s'." %
-                        (src.type, dst_type))
+                if src.type.dtype.same_as(dst_type.dtype):
+                    msg = "Memoryview '%s' not conformable to memoryview '%s'."
+                    tup = src.type, dst_type
+                else:
+                    msg = "Different base types for memoryviews (%s, %s)"
+                    tup = src.type.dtype, dst_type.dtype
+
+                error(self.pos, msg % tup)
 
         elif dst_type.is_pyobject:
             if not src.type.is_pyobject:
@@ -2412,13 +2418,13 @@ class IndexNode(ExprNode):
                 self.is_temp = True
 
             if setting and self.base.type.is_memoryviewslice:
-                self.type.writable_needed = True
+                self.base.type.writable_needed = True
             elif setting:
                 if not self.base.entry.type.writable:
                     error(self.pos, "Writing to readonly buffer")
                 else:
                     self.writable_needed = True
-                    if self.type.is_buffer:
+                    if self.base.type.is_buffer:
                         self.base.entry.buffer_aux.writable_needed = True
 
         elif memoryviewslice_access:
@@ -2570,7 +2576,7 @@ class IndexNode(ExprNode):
         if self.is_buffer_access:
             if code.globalstate.directives['nonecheck']:
                 self.put_nonecheck(code)
-            self.buffer_ptr_code = self.buffer_lookup_code(code)
+            buffer_entry, self.buffer_ptr_code = self.buffer_lookup_code(code)
             if self.type.is_pyobject:
                 # is_temp is True, so must pull out value and incref it.
                 code.putln("%s = *%s;" % (self.result(), self.buffer_ptr_code))
@@ -2657,11 +2663,14 @@ class IndexNode(ExprNode):
         # Used from generate_assignment_code and InPlaceAssignmentNode
         if code.globalstate.directives['nonecheck']:
             self.put_nonecheck(code)
-        ptrexpr = self.buffer_lookup_code(code)
+
+        buffer_entry, ptrexpr = self.buffer_lookup_code(code)
+
         if self.buffer_type.dtype.is_pyobject:
             # Must manage refcounts. Decref what is already there
             # and incref what we put in.
-            ptr = code.funcstate.allocate_temp(self.buffer_type.buffer_ptr_type, manage_ref=False)
+            ptr = code.funcstate.allocate_temp(buffer_entry.buf_ptr_type,
+                                               manage_ref=False)
             rhs_code = rhs.result()
             code.putln("%s = %s;" % (ptr, ptrexpr))
             code.put_gotref("*%s" % ptr)
@@ -2735,12 +2744,14 @@ class IndexNode(ExprNode):
             buffer_entry = MemoryView.MemoryViewSliceBufferEntry(entry)
             negative_indices = Buffer.buffer_defaults['negative_indices']
 
-        return Buffer.put_buffer_lookup_code(entry=buffer_entry,
-                                             index_signeds=[i.type.signed for i in self.indices],
-                                             index_cnames=index_temps,
-                                             directives=code.globalstate.directives,
-                                             pos=self.pos, code=code,
-                                             negative_indices=negative_indices)
+
+        return buffer_entry, Buffer.put_buffer_lookup_code(
+               entry=buffer_entry,
+               index_signeds=[i.type.signed for i in self.indices],
+               index_cnames=index_temps,
+               directives=code.globalstate.directives,
+               pos=self.pos, code=code,
+               negative_indices=negative_indices)
 
     def put_nonecheck(self, code):
         code.globalstate.use_utility_code(raise_noneindex_error_utility_code)
@@ -7681,6 +7692,13 @@ class CoerceToMemViewSliceNode(CoercionNode):
                                      self.type.from_py_function,
                                      self.arg.py_result()))
 
+        error_cond = self.type.error_condition(self.result())
+        code.putln(code.error_goto_if(error_cond, self.pos))
+
+    def generate_disposal_code(self, code):
+        code.put_xdecref_memoryviewslice(self.result(),
+                                         have_gil=not self.env.nogil)
+
 
 class CastNode(CoercionNode):
     #  Wrap a node in a C type cast.

Cython/Compiler/MemoryView.py

@@ -14,9 +14,8 @@ STOP_ERR = "Axis specification only allowed in the 'stop' slot."
 STEP_ERR = "Only the value 1 (one) or valid axis specification allowed in the step slot."
 ONE_ERR = "The value 1 (one) may appear in the first or last axis specification only."
 BOTH_CF_ERR = "Cannot specify an array that is both C and Fortran contiguous."
-NOT_AMP_ERR = "Invalid operator, only an ampersand '&' is allowed."
 INVALID_ERR = "Invalid axis specification."
-EXPR_ERR = "no expressions allowed in axis spec, only names (e.g. cython.view.contig)."
+EXPR_ERR = "no expressions allowed in axis spec, only names and literals."
 CF_ERR = "Invalid axis specification for a C/Fortran contiguous array."
 
 def concat_flags(*flags):
@@ -24,9 +23,9 @@ def concat_flags(*flags):
 
 format_flag = "PyBUF_FORMAT"
 
-memview_c_contiguous = concat_flags(format_flag, "PyBUF_C_CONTIGUOUS")
-memview_f_contiguous = concat_flags(format_flag, "PyBUF_F_CONTIGUOUS")
-memview_any_contiguous = concat_flags(format_flag, "PyBUF_ANY_CONTIGUOUS")
+memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
+memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
+memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
 memview_full_access = "PyBUF_FULL"
 #memview_strided_access = "PyBUF_STRIDED"
 memview_strided_access = "PyBUF_RECORDS"
@@ -72,8 +71,8 @@ def mangle_dtype_name(dtype):
     import Buffer
     return Buffer.mangle_dtype_name(dtype)
 
-def axes_to_str(axes):
-    return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
+#def axes_to_str(axes):
+#    return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
 
 def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code,
                                 incref_rhs=True, have_gil=False):
@@ -106,7 +105,7 @@ def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, code,
         code.putln("%s.strides[%d] = %s.strides[%d];" % (lhs_cname, i, rhs_cname, i))
         code.putln("%s.suboffsets[%d] = %s.suboffsets[%d];" % (lhs_cname, i, rhs_cname, i))
 
-def get_buf_flag(specs):
+def get_buf_flags(specs):
     is_c_contig, is_f_contig = is_cf_contig(specs)
 
     if is_c_contig:
@@ -196,50 +195,65 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
             suboffset = "%s.suboffsets[%d]" % (self.cname, dim)
             index = index_cnames[dim]
 
-            if access == 'full' and packing in ('strided', 'follow'):
+            flag = get_memoryview_flag(access, packing)
+
+            if flag == "generic":
                 code.globalstate.use_utility_code(memviewslice_index_helpers)
                 bufp = ('__pyx_memviewslice_index_full(%s, %s, %s, %s)' %
                                             (bufp, index, stride, suboffset))
 
-            elif access == 'full' and packing == 'contig':
+            elif flag == "generic_contiguous":
                 # We can skip stride multiplication with the cast
                 code.globalstate.use_utility_code(memviewslice_index_helpers)
                 bufp = '((char *) ((%s *) %s) + %s)' % (type_decl, bufp, index)
                 bufp = ('__pyx_memviewslice_index_full_contig(%s, %s)' %
                                                             (bufp, suboffset))
 
-            elif access == 'ptr' and packing in ('strided', 'follow'):
+            elif flag == "indirect":
                 bufp = ("(*((char **) %s + %s * %s) + %s)" %
                                              (bufp, index, stride, suboffset))
 
-            elif access == 'ptr' and packing == 'contig':
+            elif flag == "indirect_contiguous":
                 bufp = "(*((char **) %s) + %s)" % (bufp, suboffset)
 
-            elif access == 'direct' and packing in ('strided', 'follow'):
+            elif flag == "strided":
                 bufp = "(%s + %s * %s)" % (bufp, index, stride)
 
             else:
-                assert (access, packing) == ('direct', 'contig'), (access, packing)
+                assert flag == 'contiguous', flag
                 bufp = '((char *) (((%s *) %s) + %s))' % (type_decl, bufp, index)
 
             bufp = '( /* dim=%d */ %s )' % (dim, bufp)
 
         return "((%s *) %s)" % (type_decl, bufp)
 
+def get_memoryview_flag(access, packing):
+    if access == 'full' and packing in ('strided', 'follow'):
+        return 'generic'
+    elif access == 'full' and packing == 'contig':
+        return 'generic_contiguous'
+    elif access == 'ptr' and packing in ('strided', 'follow'):
+        return 'indirect'
+    elif access == 'ptr' and packing == 'contig':
+        return 'indirect_contiguous'
+    elif access == 'direct' and packing in ('strided', 'follow'):
+        return 'strided'
+    else:
+        assert (access, packing) == ('direct', 'contig'), (access, packing)
+        return 'contiguous'
+
 def get_copy_func_name(to_memview):
-    base = "__Pyx_BufferNew_%s_From_%s_%s"
+    base = "__Pyx_BufferNew_%s_From_%s"
     if to_memview.is_c_contig:
-        return base % ('C', axes_to_str(to_memview.axes), mangle_dtype_name(to_memview.dtype))
+        return base % ('C', to_memview.specialization_suffix())
     else:
-        return base % ('F', axes_to_str(to_memview.axes), mangle_dtype_name(to_memview.dtype))
+        return base % ('F', to_memview.specialization_suffix())
 
 def get_copy_contents_name(from_mvs, to_mvs):
-    dtype = from_mvs.dtype
-    assert dtype == to_mvs.dtype
-    return ('__Pyx_BufferCopyContents_%s_%s_%s' %
-            (axes_to_str(from_mvs.axes),
-             axes_to_str(to_mvs.axes),
-             mangle_dtype_name(dtype)))
+    assert from_mvs.dtype == to_mvs.dtype
+    return '__Pyx_BufferCopyContents_%s_to_%s' % (from_mvs.specialization_suffix(),
+                                                  to_mvs.specialization_suffix())
+
 
 class IsContigFuncUtilCode(object):
     

Cython/Compiler/Nodes.py

@@ -1428,16 +1428,20 @@ class FuncDefNode(StatNode, BlockNode):
         self.generate_argument_parsing_code(env, code)
         # If an argument is assigned to in the body, we must
         # incref it to properly keep track of refcounts.
+        is_cdef = isinstance(self, CFuncDefNode)
         for entry in lenv.arg_entries:
             if entry.type.is_pyobject:
                 if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_incref(entry)
-            # Note: all memoryviewslices are already newly acquired references
-            #       or increffed defaults!
-            #if entry.type.is_memoryviewslice:
-            #    code.put_incref_memoryviewslice(entry.cname,
-            #                                    have_gil=not lenv.nogil)
-                #code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
+
+            # Note: defaults are always increffed. For def functions, we
+            #       we aquire arguments from object converstion, so we have
+            #       new references. If we are a cdef function, we need to
+            #       incref our arguments
+            if is_cdef and entry.type.is_memoryviewslice:
+                code.put_incref_memoryviewslice(entry.cname,
+                                                have_gil=not lenv.nogil)
+
         # ----- Initialise local buffer auxiliary variables
         for entry in lenv.var_entries + lenv.arg_entries:
             if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
@@ -1567,7 +1571,7 @@ class FuncDefNode(StatNode, BlockNode):
             if entry.type.is_pyobject:
                 if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_decref(entry)
-            if entry.type.is_memoryviewslice and isinstance(self, DefNode):
+            if entry.type.is_memoryviewslice:
                 code.put_xdecref_memoryviewslice(entry.cname)
                 #code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
         if self.needs_closure:

Cython/Compiler/PyrexTypes.py

@@ -338,9 +338,7 @@ class MemoryViewSliceType(PyrexType):
     to_py_function = None
 
     exception_value = None
-    exception_check = None
-
-    utility_counter = 0
+    exception_check = True
 
     def __init__(self, base_dtype, axes):
         '''
@@ -381,7 +379,7 @@ class MemoryViewSliceType(PyrexType):
         self.dtype = base_dtype
         self.axes = axes
         self.ndim = len(axes)
-        self.flags = MemoryView.get_buf_flag(self.axes)
+        self.flags = MemoryView.get_buf_flags(self.axes)
 
         self.is_c_contig, self.is_f_contig = MemoryView.is_cf_contig(self.axes)
         assert not (self.is_c_contig and self.is_f_contig)
@@ -416,7 +414,6 @@ class MemoryViewSliceType(PyrexType):
         if self.scope is None:
 
             import Symtab, MemoryView, Options
-            from MemoryView import axes_to_str
 
             self.scope = scope = Symtab.CClassScope(
                     'mvs_class_'+self.specialization_suffix(),
@@ -504,12 +501,12 @@ class MemoryViewSliceType(PyrexType):
 
     def specialization_suffix(self):
         import MemoryView
-        return MemoryView.axes_to_str(self.axes) + '_' + MemoryView.mangle_dtype_name(self.dtype)
+        dtype_name = MemoryView.mangle_dtype_name(self.dtype)
+        return "%s_%s" % (self.axes_to_name(), dtype_name)
 
-    def global_init_code(self, entry, code):
-        code.putln("%s.data = NULL;" % entry.cname)
-        code.putln("%s.memview = NULL;" % entry.cname)
-        #code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False)
+    #def global_init_code(self, entry, code):
+    #    code.putln("%s.data = NULL;" % entry.cname)
+    #    code.putln("%s.memview = NULL;" % entry.cname)
 
     def check_for_null_code(self, cname):
         return cname + '.memview'
@@ -535,15 +532,14 @@ class MemoryViewSliceType(PyrexType):
         else:
             c_or_f_flag = "0"
 
-        # specializing through UtilityCode.specialize is not so useful as
-        # specialize on too many things to include in the function name
-        funcname = "__Pyx_PyObject_to_MemoryviewSlice_%d" % self.utility_counter
+        suffix = self.specialization_suffix()
+        funcname = "__Pyx_PyObject_to_MemoryviewSlice_" + suffix
 
         context = dict(
             MemoryView.context,
             buf_flag = self.flags,
             ndim = self.ndim,
-            axes_specs = ', '.join(self.axes_specs_to_code()),
+            axes_specs = ', '.join(self.axes_to_code()),
             dtype_typedecl = self.dtype.declaration_code(""),
             struct_nesting_depth = self.dtype.struct_nesting_depth(),
             c_or_f_flag = c_or_f_flag,
@@ -551,8 +547,6 @@ class MemoryViewSliceType(PyrexType):
         )
 
         self.from_py_function = funcname
-        MemoryViewSliceType.utility_counter += 1
-
         return True
 
     def create_to_py_utility_code(self, env):
@@ -560,15 +554,15 @@ class MemoryViewSliceType(PyrexType):
 
     def get_to_py_function(self, obj):
         return "__pyx_memoryview_fromslice(&%s, %s.memview->obj, %s, %s);" % (
-                                obj.result(), obj.result(), self.flags, self.ndim)
+                       obj.result(), obj.result(), self.flags, self.ndim)
 
-    def axes_specs_to_code(self):
+    def axes_to_code(self):
         "Return a list of code constants for each axis"
         import MemoryView
         d = MemoryView._spec_to_const
         return ["(%s | %s)" % (d[a], d[p]) for a, p in self.axes]
 
-    def axes_specs_to_name(self):
+    def axes_to_name(self):
         "Return an abbreviated name for our axes"
         import MemoryView
         d = MemoryView._spec_to_abbrev
@@ -577,6 +571,24 @@ class MemoryViewSliceType(PyrexType):
     def error_condition(self, result_code):
         return "!%s.memview" % result_code
 
+    def __str__(self):
+        import MemoryView
+
+        axes_code_list = []
+        for access, packing in self.axes:
+            flag = MemoryView.get_memoryview_flag(access, packing)
+            if flag == "strided":
+                axes_code_list.append(":")
+            else:
+                axes_code_list.append("::" + flag)
+
+        if self.dtype.is_pyobject:
+            dtype_name = self.dtype.name
+        else:
+            dtype_name = self.dtype
+
+        return "%s[%s]" % (dtype_name, ", ".join(axes_code_list))
+
 
 class BufferType(BaseType):
     #
@@ -632,6 +644,9 @@ class PyObjectType(PyrexType):
     def __repr__(self):
         return "<PyObjectType>"
 
+    def __eq__(self, other):
+        return isinstance(other, PyObjectType) and self.name == other.name
+
     def can_coerce_to_pyobject(self, env):
         return True
 

Cython/Utility/MemoryView.pyx

@@ -258,6 +258,9 @@ cdef class memoryview(object):
         bytesitem = itemp[:self.view.itemsize]
         return struct.unpack(fmt, bytesitem)
 
+    def __repr__(self):
+        return "<MemoryView of %s at 0x%x>" % (self.obj.__class__.__name__, id(self))
+
     def __str__(self):
         return "<MemoryView of %r at 0x%x>" % (self.obj, id(self))
 
@@ -283,8 +286,8 @@ cdef memoryview_cwrapper(object o, int flags):
     return memoryview(o, flags)
 
 @cname('__pyx_memoryview_fromslice')
-cdef memoryview memoryview_from_memview_cwrapper(
-        {{memviewslice_name}} *memviewslice, object orig_obj, int flags, int new_ndim):
+cdef memoryview_from_memview_cwrapper({{memviewslice_name}} *memviewslice,
+                                object orig_obj, int flags, int new_ndim):
     cdef _memoryviewslice result = _memoryviewslice(orig_obj, flags)
 
     result.from_slice = memviewslice[0]

tests/errors/memview_declarations.pyx

 # mode: error
 
 cimport cython
-from cython.view cimport contig as foo, full as bar, follow
 from cython cimport view
-biz = cython.view.contig
-foz = cython.view.full
-adict = {'view': cython.view}
-alist = [adict]
+
+
+
+
+
 
 cdef signed short[::1, ::1] both
 cdef signed short[::1, :, :, ::1] both2
@@ -17,14 +17,15 @@ cdef long long[01::1, 0x01:, '0'   :, False:] fort_contig0
 cdef signed char[1::] bad_start
 cdef unsigned long[:,:1] bad_stop
 cdef unsigned long[:,::1,:] neither_c_or_f
-cdef signed char[::1, ::view.follow & view.direct] bad_f_contig
-cdef signed char[::1, ::view.follow] bad_f_contig2
-cdef signed char[::view.contig | view.direct] not_ampersand
-cdef signed char[::view.ptr & view.direct] no_access_spec
 cdef signed char[::1-1+1] expr_spec
 cdef signed char[::blargh] bad_name
 cdef double[::alist[0]['view'].full] expr_attribute
-cdef double[::view.ptr & view.follow] no_single_follow
+
+cdef object[::1, :] unconformable1 = object()
+cdef object[:, ::1] unconformable2 = unconformable1
+
+cdef int[::1, :] dtype_unconformable = object()
+unconformable1 = dtype_unconformable
 
 _ERRORS = u'''
 11:25: Cannot specify an array that is both C and Fortran contiguous.
@@ -36,12 +37,9 @@ _ERRORS = u'''
 17:18: there must be nothing or the value 0 (zero) in the start slot.
 18:22: Axis specification only allowed in the 'stop' slot.
 19:23: The value 1 (one) may appear in the first or last axis specification only.
-20:36: Invalid axis specification for a C/Fortran contiguous array.
-21:28: Invalid axis specification for a C/Fortran contiguous array.
-22:31: Invalid operator, only an ampersand '&' is allowed.
-23:28: Invalid axis specification.
-24:22: Invalid axis specification.
-25:25: Invalid axis specification.
-26:22: no expressions allowed in axis spec, only names (e.g. cython.view.contig).
-27:12: Invalid use of the follow specifier.
+20:22: Invalid axis specification.
+21:25: Invalid axis specification.
+22:22: no expressions allowed in axis spec, only names and literals.
+25:51: Memoryview 'object[::contiguous, :]' not conformable to memoryview 'object[:, ::contiguous]'.
+28:36: Different base types for memoryviews (int, Python object)
 '''

tests/run/bufaccess.pyx

@@ -1062,22 +1062,6 @@ def nested_packed_struct(object[NestedPackedStruct] buf):
     """
     print buf[0].a, buf[0].b, buf[0].sub.a, buf[0].sub.b, buf[0].c
 
-cdef struct LongComplex:
-    long double real
-    long double imag
-
-cdef class LongComplexMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        cdef LongComplex* s
-        s = <LongComplex*>buf;
-        s.real, s.imag = value
-        return 0
-
-    cdef get_itemsize(self): return sizeof(LongComplex)
-    cdef get_default_format(self): return b"Zg"
-
-#cdef extern from "complex.h":
-#    pass
 
 @testcase
 def complex_dtype(object[long double complex] buf):

tests/run/mockbuffers.pxi

@@ -311,3 +311,16 @@ cdef class NestedPackedStructMockBuffer(MockBuffer):
     cdef get_itemsize(self): return sizeof(NestedPackedStruct)
     cdef get_default_format(self): return b"ci^ci@i"
 
+cdef struct LongComplex:
+    long double real
+    long double imag
+
+cdef class LongComplexMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        cdef LongComplex* s
+        s = <LongComplex*>buf;
+        s.real, s.imag = value
+        return 0
+
+    cdef get_itemsize(self): return sizeof(LongComplex)
+    cdef get_default_format(self): return b"Zg"