Initial attempt at implementing read-only memoryviews. (Github issue #1605)

Cython/Compiler/ExprNodes.py

@@ -870,16 +870,19 @@ class ExprNode(Node):
                 elif not src_type.is_error:
                     error(self.pos,
                           "Cannot convert '%s' to memoryviewslice" % (src_type,))
-            elif not src.type.conforms_to(dst_type, broadcast=self.is_memview_broadcast,
-                                          copying=self.is_memview_copy_assignment):
-                if src.type.dtype.same_as(dst_type.dtype):
-                    msg = "Memoryview '%s' not conformable to memoryview '%s'."
-                    tup = src.type, dst_type
-                else:
-                    msg = "Different base types for memoryviews (%s, %s)"
-                    tup = src.type.dtype, dst_type.dtype
+            else:
+                if src.type.writable_needed:
+                    dst_type.writable_needed = True
+                if not src.type.conforms_to(dst_type, broadcast=self.is_memview_broadcast,
+                                            copying=self.is_memview_copy_assignment):
+                    if src.type.dtype.same_as(dst_type.dtype):
+                        msg = "Memoryview '%s' not conformable to memoryview '%s'."
+                        tup = src.type, dst_type
+                    else:
+                        msg = "Different base types for memoryviews (%s, %s)"
+                        tup = src.type.dtype, dst_type.dtype
 
-                error(self.pos, msg % tup)
+                    error(self.pos, msg % tup)
 
         elif dst_type.is_pyobject:
             if not src.type.is_pyobject:
@@ -4253,6 +4256,10 @@ class MemoryViewIndexNode(BufferIndexNode):
         indices = self.indices
         have_slices, indices, newaxes = MemoryView.unellipsify(indices, self.base.type.ndim)
 
+        if not getting:
+            self.writable_needed = True
+            self.base.entry.type.writable_needed = True
+
         self.memslice_index = (not newaxes and len(indices) == self.base.type.ndim)
         axes = []
 
@@ -12640,12 +12647,12 @@ class CoerceToMemViewSliceNode(CoercionNode):
 
     def generate_result_code(self, code):
         self.type.create_from_py_utility_code(self.env)
-        code.putln("%s = %s(%s);" % (self.result(),
-                                     self.type.from_py_function,
-                                     self.arg.py_result()))
-
-        error_cond = self.type.error_condition(self.result())
-        code.putln(code.error_goto_if(error_cond, self.pos))
+        code.putln(self.type.from_py_call_code(
+            self.arg.py_result(),
+            self.result(),
+            self.pos,
+            code
+        ))
 
 
 class CastNode(CoercionNode):

Cython/Compiler/FusedNode.py

@@ -374,7 +374,7 @@ class FusedCFuncDefNode(StatListNode):
             coerce_from_py_func=memslice_type.from_py_function,
             dtype=dtype)
         decl_code.putln(
-            "{{memviewslice_cname}} {{coerce_from_py_func}}(object)")
+            "{{memviewslice_cname}} {{coerce_from_py_func}}(object, int)")
 
         pyx_code.context.update(
             specialized_type_name=specialized_type.specialization_string,
@@ -384,7 +384,7 @@ class FusedCFuncDefNode(StatListNode):
             u"""
                 # try {{dtype}}
                 if itemsize == -1 or itemsize == {{sizeof_dtype}}:
-                    memslice = {{coerce_from_py_func}}(arg)
+                    memslice = {{coerce_from_py_func}}(arg, 0)
                     if memslice.memview:
                         __PYX_XDEC_MEMVIEW(&memslice, 1)
                         # print 'found a match for the buffer through format parsing'

Cython/Compiler/MemoryView.py

@@ -28,12 +28,12 @@ def concat_flags(*flags):
 
 format_flag = "PyBUF_FORMAT"
 
-memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
-memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
-memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
-memview_full_access = "PyBUF_FULL"
-#memview_strided_access = "PyBUF_STRIDED"
-memview_strided_access = "PyBUF_RECORDS"
+memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT)"
+memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT)"
+memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT)"
+memview_full_access = "PyBUF_FULL_RO"
+#memview_strided_access = "PyBUF_STRIDED_RO"
+memview_strided_access = "PyBUF_RECORDS_RO"
 
 MEMVIEW_DIRECT = '__Pyx_MEMVIEW_DIRECT'
 MEMVIEW_PTR    = '__Pyx_MEMVIEW_PTR'

Cython/Compiler/Nodes.py

@@ -3698,18 +3698,12 @@ class DefNodeWrapper(FuncDefNode):
             entry = arg.entry
             code.putln("%s = %s;" % (entry.cname, item))
         else:
-            func = arg.type.from_py_function
-            if func:
+            if arg.type.from_py_function:
                 if arg.default:
                     # C-typed default arguments must be handled here
                     code.putln('if (%s) {' % item)
-                rhs = "%s(%s)" % (func, item)
-                if arg.type.is_enum:
-                    rhs = arg.type.cast_code(rhs)
-                code.putln("%s = %s; %s" % (
-                    arg.entry.cname,
-                    rhs,
-                    code.error_goto_if(arg.type.error_condition(arg.entry.cname), arg.pos)))
+                code.putln(arg.type.from_py_call_code(
+                    item, arg.entry.cname, arg.pos, code))
                 if arg.default:
                     code.putln('} else {')
                     code.putln("%s = %s;" % (
@@ -3950,17 +3944,14 @@ class DefNodeWrapper(FuncDefNode):
 
     def generate_arg_conversion_from_pyobject(self, arg, code):
         new_type = arg.type
-        func = new_type.from_py_function
         # copied from CoerceFromPyTypeNode
-        if func:
-            lhs = arg.entry.cname
-            rhs = "%s(%s)" % (func, arg.hdr_cname)
-            if new_type.is_enum:
-                rhs = PyrexTypes.typecast(new_type, PyrexTypes.c_long_type, rhs)
-            code.putln("%s = %s; %s" % (
-                lhs,
-                rhs,
-                code.error_goto_if(new_type.error_condition(arg.entry.cname), arg.pos)))
+        if new_type.from_py_function:
+            code.putln(new_type.from_py_call_code(
+                arg.hdr_cname,
+                arg.entry.cname,
+                arg.pos,
+                code,
+            ))
         else:
             error(arg.pos, "Cannot convert Python object argument to type '%s'" % new_type)
 

Cython/Compiler/Parsing.py

@@ -2481,9 +2481,12 @@ def p_c_simple_base_type(s, self_flag, nonempty, templates = None):
         error(pos, "Expected an identifier, found '%s'" % s.sy)
     if s.systring == 'const':
         s.next()
-        base_type = p_c_base_type(s,
-            self_flag = self_flag, nonempty = nonempty, templates = templates)
-        return Nodes.CConstTypeNode(pos, base_type = base_type)
+        base_type = p_c_base_type(s, self_flag=self_flag, nonempty=nonempty, templates=templates)
+        if isinstance(base_type, Nodes.MemoryViewSliceTypeNode):
+            # reverse order to avoid having to write "(const int)[:]"
+            base_type.base_type_node = Nodes.CConstTypeNode(pos, base_type=base_type.base_type_node)
+            return base_type
+        return Nodes.CConstTypeNode(pos, base_type=base_type)
     if looking_at_base_type(s):
         #print "p_c_simple_base_type: looking_at_base_type at", s.position()
         is_basic = 1

Cython/Compiler/PyrexTypes.py

@@ -314,6 +314,21 @@ class PyrexType(BaseType):
     def needs_nonecheck(self):
         return 0
 
+    def _assign_from_py_code(self, source_code, result_code, error_pos, code,
+                             from_py_function=None, error_condition=None, extra_args=None):
+        args = ', ' + ', '.join('%s' % arg for arg in extra_args) if extra_args else ''
+        convert_call = "%s(%s%s)" % (
+            from_py_function or self.from_py_function,
+            source_code,
+            args,
+        )
+        if self.is_enum:
+            convert_call = typecast(self, c_long_type, convert_call)
+        return '%s = %s; %s' % (
+            result_code,
+            convert_call,
+            code.error_goto_if(error_condition or self.error_condition(result_code), error_pos))
+
 
 def public_decl(base_code, dll_linkage):
     if dll_linkage:
@@ -491,12 +506,11 @@ class CTypedefType(BaseType):
 
     def from_py_call_code(self, source_code, result_code, error_pos, code,
                           from_py_function=None, error_condition=None):
-        if from_py_function is None:
-            from_py_function = self.from_py_function
-        if error_condition is None:
-            error_condition = self.error_condition(result_code)
         return self.typedef_base_type.from_py_call_code(
-            source_code, result_code, error_pos, code, from_py_function, error_condition)
+            source_code, result_code, error_pos, code,
+            from_py_function or self.from_py_function,
+            error_condition or self.error_condition(result_code)
+        )
 
     def overflow_check_binop(self, binop, env, const_rhs=False):
         env.use_utility_code(UtilityCode.load("Common", "Overflow.c"))
@@ -619,6 +633,7 @@ class MemoryViewSliceType(PyrexType):
 
     def same_as_resolved_type(self, other_type):
         return ((other_type.is_memoryviewslice and
+            self.writable_needed == other_type.writable_needed and
             self.dtype.same_as(other_type.dtype) and
             self.axes == other_type.axes) or
             other_type is error_type)
@@ -767,7 +782,14 @@ class MemoryViewSliceType(PyrexType):
 
         src = self
 
-        if src.dtype != dst.dtype:
+        if self.writable_needed and not dst.writable_needed:
+            return False
+
+        src_dtype, dst_dtype = src.dtype, dst.dtype
+        if dst_dtype.is_const and not src_dtype.is_const:
+            dst_dtype = dst_dtype.const_base_type
+
+        if src_dtype != dst_dtype:
             return False
 
         if src.ndim != dst.ndim:
@@ -885,11 +907,9 @@ class MemoryViewSliceType(PyrexType):
 
     def from_py_call_code(self, source_code, result_code, error_pos, code,
                           from_py_function=None, error_condition=None):
-        return '%s = %s(%s); %s' % (
-            result_code,
-            from_py_function or self.from_py_function,
-            source_code,
-            code.error_goto_if(error_condition or self.error_condition(result_code), error_pos))
+        return self._assign_from_py_code(
+            source_code, result_code, error_pos, code, from_py_function, error_condition,
+            extra_args=['PyBUF_WRITABLE' if self.writable_needed else '0'])
 
     def create_to_py_utility_code(self, env):
         self._dtype_to_py_func, self._dtype_from_py_func = self.dtype_object_conversion_funcs(env)
@@ -1467,11 +1487,9 @@ class CType(PyrexType):
 
     def from_py_call_code(self, source_code, result_code, error_pos, code,
                           from_py_function=None, error_condition=None):
-        return '%s = %s(%s); %s' % (
-            result_code,
-            from_py_function or self.from_py_function,
-            source_code,
-            code.error_goto_if(error_condition or self.error_condition(result_code), error_pos))
+        return self._assign_from_py_code(
+            source_code, result_code, error_pos, code, from_py_function, error_condition)
+
 
 class PythranExpr(CType):
     # Pythran object of a given type
@@ -2403,6 +2421,7 @@ class CArrayType(CPointerBaseType):
 
     def from_py_call_code(self, source_code, result_code, error_pos, code,
                           from_py_function=None, error_condition=None):
+        assert not error_condition
         call_code = "%s(%s, %s, %s)" % (
             from_py_function or self.from_py_function,
             source_code, result_code, self.size)
@@ -3826,16 +3845,6 @@ class CEnumType(CType):
                      "FROM_PY_FUNCTION": self.from_py_function}))
         return True
 
-    def from_py_call_code(self, source_code, result_code, error_pos, code,
-                          from_py_function=None, error_condition=None):
-        rhs = "%s(%s)" % (
-            from_py_function or self.from_py_function,
-            source_code)
-        return '%s = %s;%s' % (
-            result_code,
-            typecast(self, c_long_type, rhs),
-            ' %s' % code.error_goto_if(error_condition or self.error_condition(result_code), error_pos))
-
     def create_type_wrapper(self, env):
         from .UtilityCode import CythonUtilityCode
         env.use_utility_code(CythonUtilityCode.load(

Cython/Utility/MemoryView.pyx

@@ -65,6 +65,7 @@ cdef extern from *:
         PyBUF_STRIDES
         PyBUF_INDIRECT
         PyBUF_RECORDS
+        PyBUF_RECORDS_RO
 
     ctypedef struct __Pyx_TypeInfo:
         pass
@@ -408,6 +409,9 @@ cdef class memoryview(object):
             return self.convert_item_to_object(itemp)
 
     def __setitem__(memoryview self, object index, object value):
+        if self.view.readonly:
+            raise TypeError("Cannot assign to read-only memoryview")
+
         have_slices, index = _unellipsify(index, self.view.ndim)
 
         if have_slices:
@@ -507,6 +511,9 @@ cdef class memoryview(object):
 
     @cname('getbuffer')
     def __getbuffer__(self, Py_buffer *info, int flags):
+        if flags & PyBUF_WRITABLE and self.view.readonly:
+            raise ValueError("Cannot create writable memory view from read-only memoryview")
+
         if flags & PyBUF_STRIDES:
             info.shape = self.view.shape
         else:
@@ -531,7 +538,7 @@ cdef class memoryview(object):
         info.ndim = self.view.ndim
         info.itemsize = self.view.itemsize
         info.len = self.view.len
-        info.readonly = 0
+        info.readonly = self.view.readonly
         info.obj = self
 
     __pyx_getbuffer = capsule(<void *> &__pyx_memoryview_getbuffer, "getbuffer(obj, view, flags)")
@@ -1012,7 +1019,10 @@ cdef memoryview_fromslice({{memviewslice_name}} memviewslice,
     (<__pyx_buffer *> &result.view).obj = Py_None
     Py_INCREF(Py_None)
 
-    result.flags = PyBUF_RECORDS
+    if (<memoryview>memviewslice.memview).flags & PyBUF_WRITABLE:
+        result.flags = PyBUF_RECORDS
+    else:
+        result.flags = PyBUF_RECORDS_RO
 
     result.view.shape = <Py_ssize_t *> result.from_slice.shape
     result.view.strides = <Py_ssize_t *> result.from_slice.strides

Cython/Utility/MemoryView_C.c

@@ -82,7 +82,7 @@ typedef volatile __pyx_atomic_int_type __pyx_atomic_int;
 
 /////////////// ObjectToMemviewSlice.proto ///////////////
 
-static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *);
+static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *, int writable_flag);
 
 
 ////////// MemviewSliceInit.proto //////////
@@ -127,7 +127,7 @@ static CYTHON_INLINE char *__pyx_memviewslice_index_full(
 /////////////// ObjectToMemviewSlice ///////////////
 //@requires: MemviewSliceValidateAndInit
 
-static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
+static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj, int writable_flag) {
     {{memviewslice_name}} result = {{memslice_init}};
     __Pyx_BufFmt_StackElem stack[{{struct_nesting_depth}}];
     int axes_specs[] = { {{axes_specs}} };
@@ -140,7 +140,7 @@ static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
     }
 
     retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, {{c_or_f_flag}},
-                                                 {{buf_flag}}, {{ndim}},
+                                                 {{buf_flag}} | writable_flag, {{ndim}},
                                                  &{{dtype_typeinfo}}, stack,
                                                  &result, obj);
 

tests/buffers/bufaccess.pyx

@@ -599,7 +599,7 @@ def readonly(obj):
     acquired R
     25
     released R
-    >>> [str(x) for x in R.recieved_flags]  # Works in both py2 and py3
+    >>> [str(x) for x in R.received_flags]  # Works in both py2 and py3
     ['FORMAT', 'INDIRECT', 'ND', 'STRIDES']
     """
     cdef object[unsigned short int, ndim=3] buf = obj
@@ -612,7 +612,7 @@ def writable(obj):
     >>> writable(R)
     acquired R
     released R
-    >>> [str(x) for x in R.recieved_flags] # Py2/3
+    >>> [str(x) for x in R.received_flags] # Py2/3
     ['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE']
     """
     cdef object[unsigned short int, ndim=3] buf = obj
@@ -626,7 +626,7 @@ def strided(object[int, ndim=1, mode='strided'] buf):
     acquired A
     released A
     2
-    >>> [str(x) for x in A.recieved_flags] # Py2/3
+    >>> [str(x) for x in A.received_flags] # Py2/3
     ['FORMAT', 'ND', 'STRIDES']
 
     Check that the suboffsets were patched back prior to release.
@@ -641,7 +641,7 @@ def c_contig(object[int, ndim=1, mode='c'] buf):
     >>> A = IntMockBuffer(None, range(4))
     >>> c_contig(A)
     2
-    >>> [str(x) for x in A.recieved_flags]
+    >>> [str(x) for x in A.received_flags]
     ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS']
     """
     return buf[2]
@@ -654,7 +654,7 @@ def c_contig_2d(object[int, ndim=2, mode='c'] buf):
     >>> A = IntMockBuffer(None, range(12), shape=(3,4))
     >>> c_contig_2d(A)
     7
-    >>> [str(x) for x in A.recieved_flags]
+    >>> [str(x) for x in A.received_flags]
     ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS']
     """
     return buf[1, 3]
@@ -665,7 +665,7 @@ def f_contig(object[int, ndim=1, mode='fortran'] buf):
     >>> A = IntMockBuffer(None, range(4))
     >>> f_contig(A)
     2
-    >>> [str(x) for x in A.recieved_flags]
+    >>> [str(x) for x in A.received_flags]
     ['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS']
     """
     return buf[2]
@@ -678,7 +678,7 @@ def f_contig_2d(object[int, ndim=2, mode='fortran'] buf):
     >>> A = IntMockBuffer(None, range(12), shape=(4,3), strides=(1, 4))
     >>> f_contig_2d(A)
     7
-    >>> [str(x) for x in A.recieved_flags]
+    >>> [str(x) for x in A.received_flags]
     ['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS']
     """
     return buf[3, 1]
@@ -1103,7 +1103,7 @@ def bufdefaults1(IntStridedMockBuffer[int, ndim=1] buf):
     >>> bufdefaults1(A)
     acquired A
     released A
-    >>> [str(x) for x in A.recieved_flags]
+    >>> [str(x) for x in A.received_flags]
     ['FORMAT', 'ND', 'STRIDES']
     """
     pass

tests/buffers/mockbuffers.pxi

@@ -18,16 +18,17 @@ cdef class MockBuffer:
     cdef object format, offset
     cdef void* buffer
     cdef Py_ssize_t len, itemsize
-    cdef int ndim
     cdef Py_ssize_t* strides
     cdef Py_ssize_t* shape
     cdef Py_ssize_t* suboffsets
     cdef object label, log
+    cdef int ndim
+    cdef bint writable
 
-    cdef readonly object recieved_flags, release_ok
+    cdef readonly object received_flags, release_ok
     cdef public object fail
 
-    def __init__(self, label, data, shape=None, strides=None, format=None, offset=0):
+    def __init__(self, label, data, shape=None, strides=None, format=None, writable=True, offset=0):
         # It is important not to store references to data after the constructor
         # as refcounting is checked on object buffers.
         self.label = label
@@ -35,6 +36,7 @@ cdef class MockBuffer:
         self.log = ""
         self.offset = offset
         self.itemsize = self.get_itemsize()
+        self.writable = writable
         if format is None: format = self.get_default_format()
         if shape is None: shape = (len(data),)
         if strides is None:
@@ -127,16 +129,19 @@ cdef class MockBuffer:
         if self.fail:
             raise ValueError("Failing on purpose")
 
-        self.recieved_flags = []
+        self.received_flags = []
         cdef int value
         for name, value in available_flags:
             if (value & flags) == value:
-                self.recieved_flags.append(name)
+                self.received_flags.append(name)
+
+        if flags & cpython.buffer.PyBUF_WRITABLE and not self.writable:
+            raise BufferError("Writable buffer requested from read-only mock: %s" % ' | '.join(self.received_flags))
 
         buffer.buf = <void*>(<char*>self.buffer + (<int>self.offset * self.itemsize))
         buffer.obj = self
         buffer.len = self.len
-        buffer.readonly = 0
+        buffer.readonly = not self.writable
         buffer.format = <char*>self.format
         buffer.ndim = self.ndim
         buffer.shape = self.shape

tests/memoryview/memoryview.pyx

@@ -420,7 +420,7 @@ def writable(unsigned short int[:, :, :] mslice):
     >>> writable(R)
     acquired R
     released R
-    >>> [str(x) for x in R.recieved_flags] # Py2/3
+    >>> [str(x) for x in R.received_flags] # Py2/3
     ['FORMAT', 'ND', 'STRIDES', 'WRITABLE']
     """
     buf = mslice