Skip to content

C
cython
Commit 1800cbda authored by Stefan Behnel's avatar Stefan Behnel
Browse files
Options

Initial attempt at implementing read-only memoryviews. (Github issue #1605)

parent 1487db2f
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 215 additions and 173 deletions
Cython/Compiler/ExprNodes.py
View file @ 1800cbda
......@@ -870,16 +870,19 @@ class ExprNode(Node):
elif not src_type.is_error:
error(self.pos,
"Cannot convert '%s' to memoryviewslice" % (src_type,))
elif not src.type.conforms_to(dst_type, broadcast=self.is_memview_broadcast,
copying=self.is_memview_copy_assignment):
if src.type.dtype.same_as(dst_type.dtype):
msg = "Memoryview '%s' not conformable to memoryview '%s'."
tup = src.type, dst_type
else:
msg = "Different base types for memoryviews (%s, %s)"
tup = src.type.dtype, dst_type.dtype
else:
if src.type.writable_needed:
dst_type.writable_needed = True
if not src.type.conforms_to(dst_type, broadcast=self.is_memview_broadcast,
copying=self.is_memview_copy_assignment):
if src.type.dtype.same_as(dst_type.dtype):
msg = "Memoryview '%s' not conformable to memoryview '%s'."
tup = src.type, dst_type
else:
msg = "Different base types for memoryviews (%s, %s)"
tup = src.type.dtype, dst_type.dtype
error(self.pos, msg % tup)
error(self.pos, msg % tup)
elif dst_type.is_pyobject:
if not src.type.is_pyobject:
......@@ -4253,6 +4256,10 @@ class MemoryViewIndexNode(BufferIndexNode):
indices = self.indices
have_slices, indices, newaxes = MemoryView.unellipsify(indices, self.base.type.ndim)
if not getting:
self.writable_needed = True
self.base.entry.type.writable_needed = True
self.memslice_index = (not newaxes and len(indices) == self.base.type.ndim)
axes = []
......@@ -12640,12 +12647,12 @@ class CoerceToMemViewSliceNode(CoercionNode):
def generate_result_code(self, code):
self.type.create_from_py_utility_code(self.env)
code.putln("%s = %s(%s);" % (self.result(),
self.type.from_py_function,
self.arg.py_result()))
error_cond = self.type.error_condition(self.result())
code.putln(code.error_goto_if(error_cond, self.pos))
code.putln(self.type.from_py_call_code(
self.arg.py_result(),
self.result(),
self.pos,
code
))
class CastNode(CoercionNode):
......
Cython/Compiler/FusedNode.py
View file @ 1800cbda
......@@ -374,7 +374,7 @@ class FusedCFuncDefNode(StatListNode):
coerce_from_py_func=memslice_type.from_py_function,
dtype=dtype)
decl_code.putln(
"{{memviewslice_cname}} {{coerce_from_py_func}}(object)")
"{{memviewslice_cname}} {{coerce_from_py_func}}(object, int)")
pyx_code.context.update(
specialized_type_name=specialized_type.specialization_string,
......@@ -384,7 +384,7 @@ class FusedCFuncDefNode(StatListNode):
u"""
# try {{dtype}}
if itemsize == -1 or itemsize == {{sizeof_dtype}}:
memslice = {{coerce_from_py_func}}(arg)
memslice = {{coerce_from_py_func}}(arg, 0)
if memslice.memview:
__PYX_XDEC_MEMVIEW(&memslice, 1)
# print 'found a match for the buffer through format parsing'
......
Cython/Compiler/MemoryView.py
View file @ 1800cbda
......@@ -28,12 +28,12 @@ def concat_flags(*flags):
format_flag = "PyBUF_FORMAT"
memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
memview_full_access = "PyBUF_FULL"
#memview_strided_access = "PyBUF_STRIDED"
memview_strided_access = "PyBUF_RECORDS"
memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT)"
memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT)"
memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT)"
memview_full_access = "PyBUF_FULL_RO"
#memview_strided_access = "PyBUF_STRIDED_RO"
memview_strided_access = "PyBUF_RECORDS_RO"
MEMVIEW_DIRECT = '__Pyx_MEMVIEW_DIRECT'
MEMVIEW_PTR = '__Pyx_MEMVIEW_PTR'
......
Cython/Compiler/Nodes.py
View file @ 1800cbda
......@@ -3698,18 +3698,12 @@ class DefNodeWrapper(FuncDefNode):
entry = arg.entry
code.putln("%s = %s;" % (entry.cname, item))
else:
func = arg.type.from_py_function
if func:
if arg.type.from_py_function:
if arg.default:
# C-typed default arguments must be handled here
code.putln('if (%s) {' % item)
rhs = "%s(%s)" % (func, item)
if arg.type.is_enum:
rhs = arg.type.cast_code(rhs)
code.putln("%s = %s; %s" % (
arg.entry.cname,
rhs,
code.error_goto_if(arg.type.error_condition(arg.entry.cname), arg.pos)))
code.putln(arg.type.from_py_call_code(
item, arg.entry.cname, arg.pos, code))
if arg.default:
code.putln('} else {')
code.putln("%s = %s;" % (
......@@ -3950,17 +3944,14 @@ class DefNodeWrapper(FuncDefNode):
def generate_arg_conversion_from_pyobject(self, arg, code):
new_type = arg.type
func = new_type.from_py_function
# copied from CoerceFromPyTypeNode
if func:
lhs = arg.entry.cname
rhs = "%s(%s)" % (func, arg.hdr_cname)
if new_type.is_enum:
rhs = PyrexTypes.typecast(new_type, PyrexTypes.c_long_type, rhs)
code.putln("%s = %s; %s" % (
lhs,
rhs,
code.error_goto_if(new_type.error_condition(arg.entry.cname), arg.pos)))
if new_type.from_py_function:
code.putln(new_type.from_py_call_code(
arg.hdr_cname,
arg.entry.cname,
arg.pos,
code,
))
else:
error(arg.pos, "Cannot convert Python object argument to type '%s'" % new_type)
......
Cython/Compiler/Parsing.py
View file @ 1800cbda
......@@ -2481,9 +2481,12 @@ def p_c_simple_base_type(s, self_flag, nonempty, templates = None):
error(pos, "Expected an identifier, found '%s'" % s.sy)
if s.systring == 'const':
s.next()
base_type = p_c_base_type(s,
self_flag = self_flag, nonempty = nonempty, templates = templates)
return Nodes.CConstTypeNode(pos, base_type = base_type)
base_type = p_c_base_type(s, self_flag=self_flag, nonempty=nonempty, templates=templates)
if isinstance(base_type, Nodes.MemoryViewSliceTypeNode):
# reverse order to avoid having to write "(const int)[:]"
base_type.base_type_node = Nodes.CConstTypeNode(pos, base_type=base_type.base_type_node)
return base_type
return Nodes.CConstTypeNode(pos, base_type=base_type)
if looking_at_base_type(s):
#print "p_c_simple_base_type: looking_at_base_type at", s.position()
is_basic = 1
......
Cython/Compiler/PyrexTypes.py
View file @ 1800cbda
......@@ -314,6 +314,21 @@ class PyrexType(BaseType):
def needs_nonecheck(self):
return 0
def _assign_from_py_code(self, source_code, result_code, error_pos, code,
from_py_function=None, error_condition=None, extra_args=None):
args = ', ' + ', '.join('%s' % arg for arg in extra_args) if extra_args else ''
convert_call = "%s(%s%s)" % (
from_py_function or self.from_py_function,
source_code,
args,
)
if self.is_enum:
convert_call = typecast(self, c_long_type, convert_call)
return '%s = %s; %s' % (
result_code,
convert_call,
code.error_goto_if(error_condition or self.error_condition(result_code), error_pos))
def public_decl(base_code, dll_linkage):
if dll_linkage:
......@@ -491,12 +506,11 @@ class CTypedefType(BaseType):
def from_py_call_code(self, source_code, result_code, error_pos, code,
from_py_function=None, error_condition=None):
if from_py_function is None:
from_py_function = self.from_py_function
if error_condition is None:
error_condition = self.error_condition(result_code)
return self.typedef_base_type.from_py_call_code(
source_code, result_code, error_pos, code, from_py_function, error_condition)
source_code, result_code, error_pos, code,
from_py_function or self.from_py_function,
error_condition or self.error_condition(result_code)
)
def overflow_check_binop(self, binop, env, const_rhs=False):
env.use_utility_code(UtilityCode.load("Common", "Overflow.c"))
......@@ -619,6 +633,7 @@ class MemoryViewSliceType(PyrexType):
def same_as_resolved_type(self, other_type):
return ((other_type.is_memoryviewslice and
self.writable_needed == other_type.writable_needed and
self.dtype.same_as(other_type.dtype) and
self.axes == other_type.axes) or
other_type is error_type)
......@@ -767,7 +782,14 @@ class MemoryViewSliceType(PyrexType):
src = self
if src.dtype != dst.dtype:
if self.writable_needed and not dst.writable_needed:
return False
src_dtype, dst_dtype = src.dtype, dst.dtype
if dst_dtype.is_const and not src_dtype.is_const:
dst_dtype = dst_dtype.const_base_type
if src_dtype != dst_dtype:
return False
if src.ndim != dst.ndim:
......@@ -885,11 +907,9 @@ class MemoryViewSliceType(PyrexType):
def from_py_call_code(self, source_code, result_code, error_pos, code,
from_py_function=None, error_condition=None):
return '%s = %s(%s); %s' % (
result_code,
from_py_function or self.from_py_function,
source_code,
code.error_goto_if(error_condition or self.error_condition(result_code), error_pos))
return self._assign_from_py_code(
source_code, result_code, error_pos, code, from_py_function, error_condition,
extra_args=['PyBUF_WRITABLE' if self.writable_needed else '0'])
def create_to_py_utility_code(self, env):
self._dtype_to_py_func, self._dtype_from_py_func = self.dtype_object_conversion_funcs(env)
......@@ -1467,11 +1487,9 @@ class CType(PyrexType):
def from_py_call_code(self, source_code, result_code, error_pos, code,
from_py_function=None, error_condition=None):
return '%s = %s(%s); %s' % (
result_code,
from_py_function or self.from_py_function,
source_code,
code.error_goto_if(error_condition or self.error_condition(result_code), error_pos))
return self._assign_from_py_code(
source_code, result_code, error_pos, code, from_py_function, error_condition)
class PythranExpr(CType):
# Pythran object of a given type
......@@ -2403,6 +2421,7 @@ class CArrayType(CPointerBaseType):
def from_py_call_code(self, source_code, result_code, error_pos, code,
from_py_function=None, error_condition=None):
assert not error_condition
call_code = "%s(%s, %s, %s)" % (
from_py_function or self.from_py_function,
source_code, result_code, self.size)
......@@ -3826,16 +3845,6 @@ class CEnumType(CType):
"FROM_PY_FUNCTION": self.from_py_function}))
return True
def from_py_call_code(self, source_code, result_code, error_pos, code,
from_py_function=None, error_condition=None):
rhs = "%s(%s)" % (
from_py_function or self.from_py_function,
source_code)
return '%s = %s;%s' % (
result_code,
typecast(self, c_long_type, rhs),
' %s' % code.error_goto_if(error_condition or self.error_condition(result_code), error_pos))
def create_type_wrapper(self, env):
from .UtilityCode import CythonUtilityCode
env.use_utility_code(CythonUtilityCode.load(
......
Cython/Utility/MemoryView.pyx
View file @ 1800cbda
......@@ -65,6 +65,7 @@ cdef extern from *:
PyBUF_STRIDES
PyBUF_INDIRECT
PyBUF_RECORDS
PyBUF_RECORDS_RO
ctypedef struct __Pyx_TypeInfo:
pass
......@@ -408,6 +409,9 @@ cdef class memoryview(object):
return self.convert_item_to_object(itemp)
def __setitem__(memoryview self, object index, object value):
if self.view.readonly:
raise TypeError("Cannot assign to read-only memoryview")
have_slices, index = _unellipsify(index, self.view.ndim)
if have_slices:
......@@ -507,6 +511,9 @@ cdef class memoryview(object):
@cname('getbuffer')
def __getbuffer__(self, Py_buffer *info, int flags):
if flags & PyBUF_WRITABLE and self.view.readonly:
raise ValueError("Cannot create writable memory view from read-only memoryview")
if flags & PyBUF_STRIDES:
info.shape = self.view.shape
else:
......@@ -531,7 +538,7 @@ cdef class memoryview(object):
info.ndim = self.view.ndim
info.itemsize = self.view.itemsize
info.len = self.view.len
info.readonly = 0
info.readonly = self.view.readonly
info.obj = self
__pyx_getbuffer = capsule(<void *> &__pyx_memoryview_getbuffer, "getbuffer(obj, view, flags)")
......@@ -1012,7 +1019,10 @@ cdef memoryview_fromslice({{memviewslice_name}} memviewslice,
(<__pyx_buffer *> &result.view).obj = Py_None
Py_INCREF(Py_None)
result.flags = PyBUF_RECORDS
if (<memoryview>memviewslice.memview).flags & PyBUF_WRITABLE:
result.flags = PyBUF_RECORDS
else:
result.flags = PyBUF_RECORDS_RO
result.view.shape = <Py_ssize_t *> result.from_slice.shape
result.view.strides = <Py_ssize_t *> result.from_slice.strides
......
Cython/Utility/MemoryView_C.c
View file @ 1800cbda
......@@ -82,7 +82,7 @@ typedef volatile __pyx_atomic_int_type __pyx_atomic_int;
/////////////// ObjectToMemviewSlice.proto ///////////////
static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *);
static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *, int writable_flag);
////////// MemviewSliceInit.proto //////////
......@@ -127,7 +127,7 @@ static CYTHON_INLINE char *__pyx_memviewslice_index_full(
/////////////// ObjectToMemviewSlice ///////////////
//@requires: MemviewSliceValidateAndInit
static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj, int writable_flag) {
{{memviewslice_name}} result = {{memslice_init}};
__Pyx_BufFmt_StackElem stack[{{struct_nesting_depth}}];
int axes_specs[] = { {{axes_specs}} };
......@@ -140,7 +140,7 @@ static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
}
retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, {{c_or_f_flag}},
{{buf_flag}}, {{ndim}},
{{buf_flag}} | writable_flag, {{ndim}},
&{{dtype_typeinfo}}, stack,
&result, obj);
......
tests/buffers/bufaccess.pyx
View file @ 1800cbda
......@@ -599,7 +599,7 @@ def readonly(obj):
acquired R
25
released R
>>> [str(x) for x in R.recieved_flags] # Works in both py2 and py3
>>> [str(x) for x in R.received_flags] # Works in both py2 and py3
['FORMAT', 'INDIRECT', 'ND', 'STRIDES']
"""
cdef object[unsigned short int, ndim=3] buf = obj
......@@ -612,7 +612,7 @@ def writable(obj):
>>> writable(R)
acquired R
released R
>>> [str(x) for x in R.recieved_flags] # Py2/3
>>> [str(x) for x in R.received_flags] # Py2/3
['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE']
"""
cdef object[unsigned short int, ndim=3] buf = obj
......@@ -626,7 +626,7 @@ def strided(object[int, ndim=1, mode='strided'] buf):
acquired A
released A
2
>>> [str(x) for x in A.recieved_flags] # Py2/3
>>> [str(x) for x in A.received_flags] # Py2/3
['FORMAT', 'ND', 'STRIDES']
Check that the suboffsets were patched back prior to release.
......@@ -641,7 +641,7 @@ def c_contig(object[int, ndim=1, mode='c'] buf):
>>> A = IntMockBuffer(None, range(4))
>>> c_contig(A)
2
>>> [str(x) for x in A.recieved_flags]
>>> [str(x) for x in A.received_flags]
['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS']
"""
return buf[2]
......@@ -654,7 +654,7 @@ def c_contig_2d(object[int, ndim=2, mode='c'] buf):
>>> A = IntMockBuffer(None, range(12), shape=(3,4))
>>> c_contig_2d(A)
7
>>> [str(x) for x in A.recieved_flags]
>>> [str(x) for x in A.received_flags]
['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS']
"""
return buf[1, 3]
......@@ -665,7 +665,7 @@ def f_contig(object[int, ndim=1, mode='fortran'] buf):
>>> A = IntMockBuffer(None, range(4))
>>> f_contig(A)
2
>>> [str(x) for x in A.recieved_flags]
>>> [str(x) for x in A.received_flags]
['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS']
"""
return buf[2]
......@@ -678,7 +678,7 @@ def f_contig_2d(object[int, ndim=2, mode='fortran'] buf):
>>> A = IntMockBuffer(None, range(12), shape=(4,3), strides=(1, 4))
>>> f_contig_2d(A)
7
>>> [str(x) for x in A.recieved_flags]
>>> [str(x) for x in A.received_flags]
['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS']
"""
return buf[3, 1]
......@@ -1103,7 +1103,7 @@ def bufdefaults1(IntStridedMockBuffer[int, ndim=1] buf):
>>> bufdefaults1(A)
acquired A
released A
>>> [str(x) for x in A.recieved_flags]
>>> [str(x) for x in A.received_flags]
['FORMAT', 'ND', 'STRIDES']
"""
pass
......
tests/buffers/mockbuffers.pxi
View file @ 1800cbda
......@@ -18,16 +18,17 @@ cdef class MockBuffer:
cdef object format, offset
cdef void* buffer
cdef Py_ssize_t len, itemsize
cdef int ndim
cdef Py_ssize_t* strides
cdef Py_ssize_t* shape
cdef Py_ssize_t* suboffsets
cdef object label, log
cdef int ndim
cdef bint writable
cdef readonly object recieved_flags, release_ok
cdef readonly object received_flags, release_ok
cdef public object fail
def __init__(self, label, data, shape=None, strides=None, format=None, offset=0):
def __init__(self, label, data, shape=None, strides=None, format=None, writable=True, offset=0):
# It is important not to store references to data after the constructor
# as refcounting is checked on object buffers.
self.label = label
......@@ -35,6 +36,7 @@ cdef class MockBuffer:
self.log = ""
self.offset = offset
self.itemsize = self.get_itemsize()
self.writable = writable
if format is None: format = self.get_default_format()
if shape is None: shape = (len(data),)
if strides is None:
......@@ -127,16 +129,19 @@ cdef class MockBuffer:
if self.fail:
raise ValueError("Failing on purpose")
self.recieved_flags = []
self.received_flags = []
cdef int value
for name, value in available_flags:
if (value & flags) == value:
self.recieved_flags.append(name)
self.received_flags.append(name)
if flags & cpython.buffer.PyBUF_WRITABLE and not self.writable:
raise BufferError("Writable buffer requested from read-only mock: %s" % ' | '.join(self.received_flags))
buffer.buf = <void*>(<char*>self.buffer + (<int>self.offset * self.itemsize))
buffer.obj = self
buffer.len = self.len
buffer.readonly = 0
buffer.readonly = not self.writable
buffer.format = <char*>self.format
buffer.ndim = self.ndim
buffer.shape = self.shape
......
tests/memoryview/memoryview.pyx
View file @ 1800cbda
......@@ -420,7 +420,7 @@ def writable(unsigned short int[:, :, :] mslice):
>>> writable(R)
acquired R
released R
>>> [str(x) for x in R.recieved_flags] # Py2/3
>>> [str(x) for x in R.received_flags] # Py2/3
['FORMAT', 'ND', 'STRIDES', 'WRITABLE']
"""
buf = mslice
......
tests/memoryview/memslice.pyx
View file @ 1800cbda
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7