Skip to content

C
cython
Commit 3693a7b8 authored by Mark Florisson's avatar Mark Florisson
Browse files
Options

Support copying/slice assigning with dtype object

parent 056e3310
Show whitespace changes
Inline Side-by-side
Showing with 176 additions and 53 deletions
Cython/Compiler/MemoryView.py
View file @ 3693a7b8
......@@ -441,9 +441,10 @@ def copy_broadcast_memview_src_to_dst(src, dst, code):
verify_direct_dimensions(dst)
code.putln(code.error_goto_if_neg(
"%s(%s, %s, %d, %d)" % (copy_src_to_dst_cname(),
"%s(%s, %s, %d, %d, %d)" % (copy_src_to_dst_cname(),
src.result(), dst.result(),
src.type.ndim, dst.type.ndim),
src.type.ndim, dst.type.ndim,
dst.type.dtype.is_pyobject),
dst.pos))
def copy_c_or_fortran_cname(memview):
......@@ -483,7 +484,8 @@ def get_copy_new_utility(pos, from_memview, to_memview):
dtype_decl=to_memview.dtype.declaration_code(''),
contig_flag=contig_flag,
ndim=to_memview.ndim,
func_cname=copy_c_or_fortran_cname(to_memview)),
func_cname=copy_c_or_fortran_cname(to_memview),
dtype_is_object=int(to_memview.dtype.is_pyobject)),
requires=[copy_contents_new_utility])
def get_axes_specs(env, axes):
......
Cython/Compiler/PyrexTypes.py
View file @ 3693a7b8
......@@ -643,8 +643,9 @@ class MemoryViewSliceType(PyrexType):
to_py_func = "(PyObject *(*)(char *)) " + to_py_func
from_py_func = "(int (*)(char *, PyObject *)) " + from_py_func
tup = (obj.result(), self.ndim, to_py_func, from_py_func)
return "__pyx_memoryview_fromslice(&%s, %s, %s, %s);" % tup
tup = (obj.result(), self.ndim, to_py_func, from_py_func,
self.dtype.is_pyobject)
return "__pyx_memoryview_fromslice(&%s, %s, %s, %s, %d);" % tup
def dtype_object_conversion_funcs(self, env):
import MemoryView, Code
......
Cython/Utility/MemoryView.pyx
View file @ 3693a7b8
......@@ -14,15 +14,16 @@ cdef extern from "Python.h":
PyBUF_WRITABLE
void Py_INCREF(object)
void Py_DECREF(object)
cdef extern from *:
object __pyx_memoryview_new(object obj, int flags)
object __pyx_memoryview_new(object obj, int flags, bint dtype_is_object)
Py_ssize_t fill_contig_strides_array "__pyx_fill_contig_strides_array" (
Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t stride,
int ndim, char order) nogil
cdef void refcount_objects_in_slice "__pyx_memoryview_refcount_objects_in_slice" (
char *data, Py_ssize_t *shape, Py_ssize_t *strides, int ndim, bint inc)
@cname("__pyx_array")
cdef class array:
......@@ -39,8 +40,9 @@ cdef class array:
void (*callback_free_data)(void *data)
# cdef object _memview
cdef bint free_data
cdef bint dtype_is_object
def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format,
def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None,
mode=u"c", bint allocate_buffer=True):
self.ndim = len(shape)
......@@ -99,6 +101,8 @@ cdef class array:
if not self.data:
raise MemoryError("unable to allocate array data.")
self.dtype_is_object = format == 'O'
def __getbuffer__(self, Py_buffer *info, int flags):
cdef int bufmode = -1
if self.mode == b"c":
......@@ -127,20 +131,13 @@ cdef class array:
if self.callback_free_data != NULL:
self.callback_free_data(self.data)
elif self.free_data:
if self.dtype_is_object:
refcount_objects_in_slice(self.data, self._shape,
self._strides, self.ndim, False)
free(self.data)
self.data = NULL
if self._strides:
free(self._strides)
self._strides = NULL
if self._shape:
free(self._shape)
self._shape = NULL
self.format = NULL
self.itemsize = 0
property memview:
@cname('__pyx_cython_array_get_memview')
......@@ -152,7 +149,7 @@ cdef class array:
#return self._memview
flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE
return __pyx_memoryview_new(self, flags)
return __pyx_memoryview_new(self, flags, self.dtype_is_object)
def __getattr__(self, attr):
......@@ -166,12 +163,15 @@ cdef class array:
@cname("__pyx_array_new")
cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode, char *buf):
cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format,
char *mode, char *buf):
cdef array result
if buf == NULL:
result = array(shape, itemsize, format, mode.decode('ASCII'))
else:
result = array(shape, itemsize, format, mode.decode('ASCII'), allocate_buffer=False)
result = array(shape, itemsize, format, mode.decode('ASCII'),
allocate_buffer=False)
result.data = buf
return result
......@@ -201,11 +201,9 @@ cdef extern from *:
int __Pyx_GetBuffer(object, Py_buffer *, int) except -1
void __Pyx_ReleaseBuffer(Py_buffer *)
void Py_INCREF(object)
void Py_DECREF(object)
void Py_XINCREF(object)
ctypedef struct PyObject
void Py_INCREF(PyObject *)
void Py_DECREF(PyObject *)
cdef struct __pyx_memoryview "__pyx_memoryview_obj":
Py_buffer view
......@@ -241,7 +239,8 @@ cdef extern from *:
{{memviewslice_name}} slice_copy_contig "__pyx_memoryview_copy_new_contig"(
__Pyx_memviewslice *from_mvs,
char *mode, int ndim,
size_t sizeof_dtype, int contig_flag) nogil except *
size_t sizeof_dtype, int contig_flag,
bint dtype_is_object) nogil except *
bint slice_is_contig "__pyx_memviewslice_is_contig" (
{{memviewslice_name}} *mvs, char order, int ndim) nogil
bint slices_overlap "__pyx_slices_overlap" ({{memviewslice_name}} *slice1,
......@@ -299,20 +298,26 @@ cdef class memoryview(object):
cdef __pyx_atomic_int acquisition_count
cdef Py_buffer view
cdef int flags
cdef bint dtype_is_object
def __cinit__(memoryview self, object obj, int flags):
def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False):
self.obj = obj
self.flags = flags
if type(self) is memoryview or obj is not None:
__Pyx_GetBuffer(obj, &self.view, flags)
if <PyObject *> self.view.obj == NULL:
(<__pyx_buffer *> &self.view).obj = Py_None
Py_INCREF(None)
Py_INCREF(Py_None)
self.lock = PyThread_allocate_lock()
if self.lock == NULL:
raise MemoryError
if flags & PyBUF_FORMAT:
self.dtype_is_object = self.view.format == b'O'
else:
self.dtype_is_object = dtype_is_object
def __dealloc__(memoryview self):
if self.obj is not None:
__Pyx_ReleaseBuffer(&self.view)
......@@ -359,7 +364,8 @@ cdef class memoryview(object):
cdef is_slice(self, obj):
if not isinstance(obj, memoryview):
try:
obj = memoryview(obj, self.flags|PyBUF_ANY_CONTIGUOUS)
obj = memoryview(obj, self.flags|PyBUF_ANY_CONTIGUOUS,
self.dtype_is_object)
except TypeError:
return None
......@@ -372,7 +378,7 @@ cdef class memoryview(object):
dst = self[index]
memoryview_copy_contents(get_slice_from_memview(src, &src_slice)[0],
get_slice_from_memview(dst, &dst_slice)[0],
src.ndim, dst.ndim)
src.ndim, dst.ndim, self.dtype_is_object)
cdef setitem_slice_assign_scalar(self, index, value):
raise ValueError("Scalar assignment currently unsupported")
......@@ -531,25 +537,31 @@ cdef class memoryview(object):
def copy(self):
cdef {{memviewslice_name}} mslice
cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS
slice_copy(self, &mslice)
mslice = slice_copy_contig(&mslice, "c", self.view.ndim,
self.view.itemsize,
flags|PyBUF_C_CONTIGUOUS)
flags|PyBUF_C_CONTIGUOUS,
self.dtype_is_object)
return memoryview_copy_from_slice(self, &mslice)
def copy_fortran(self):
cdef {{memviewslice_name}} src, dst
cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS
slice_copy(self, &src)
dst = slice_copy_contig(&src, "fortran", self.view.ndim,
self.view.itemsize,
flags|PyBUF_F_CONTIGUOUS)
flags|PyBUF_F_CONTIGUOUS,
self.dtype_is_object)
return memoryview_copy_from_slice(self, &dst)
@cname('__pyx_memoryview_new')
cdef memoryview_cwrapper(object o, int flags):
return memoryview(o, flags)
cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object):
return memoryview(o, flags, dtype_is_object)
cdef tuple _unellipsify(object index, int ndim):
"""
......@@ -647,9 +659,11 @@ cdef memoryview memview_slice(memoryview memview, object indices):
if isinstance(memview, _memoryviewslice):
return memoryview_fromslice(&dst, new_ndim,
memviewsliceobj.to_object_func,
memviewsliceobj.to_dtype_func)
memviewsliceobj.to_dtype_func,
memview.dtype_is_object)
else:
return memoryview_fromslice(&dst, new_ndim, NULL, NULL)
return memoryview_fromslice(&dst, new_ndim, NULL, NULL,
memview.dtype_is_object)
#
......@@ -877,7 +891,8 @@ cdef class _memoryviewslice(memoryview):
cdef memoryview_fromslice({{memviewslice_name}} *memviewslice,
int ndim,
object (*to_object_func)(char *),
int (*to_dtype_func)(char *, object) except 0):
int (*to_dtype_func)(char *, object) except 0,
bint dtype_is_object):
cdef _memoryviewslice result
cdef int i
......@@ -885,7 +900,7 @@ cdef memoryview_fromslice({{memviewslice_name}} *memviewslice,
# assert 0 < ndim <= memviewslice.memview.view.ndim, (
# ndim, memviewslice.memview.view.ndim)
result = _memoryviewslice(None, 0)
result = _memoryviewslice(None, 0, dtype_is_object)
result.from_slice = memviewslice[0]
__PYX_INC_MEMVIEW(memviewslice, 1)
......@@ -896,7 +911,7 @@ cdef memoryview_fromslice({{memviewslice_name}} *memviewslice,
result.view.buf = <void *> memviewslice.data
result.view.ndim = ndim
(<__pyx_buffer *> &result.view).obj = Py_None
Py_INCREF(None)
Py_INCREF(Py_None)
result.flags = PyBUF_RECORDS
......@@ -959,7 +974,8 @@ cdef memoryview_copy_from_slice(memoryview memview, {{memviewslice_name}} *memvi
to_dtype_func = NULL
return memoryview_fromslice(memviewslice, memview.view.ndim,
to_object_func, to_dtype_func)
to_object_func, to_dtype_func,
memview.dtype_is_object)
#
......@@ -1127,7 +1143,8 @@ cdef int _err(object error, char *msg) except -1 with gil:
@cname('__pyx_memoryview_copy_contents')
cdef int memoryview_copy_contents({{memviewslice_name}} src,
{{memviewslice_name}} dst,
int src_ndim, int dst_ndim) nogil except -1:
int src_ndim, int dst_ndim,
bint dtype_is_object) nogil except -1:
"""
Copy memory from slice src to slice dst.
Check for overlapping memory and verify the shapes.
......@@ -1176,7 +1193,9 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src,
if direct_copy:
# Contiguous slices with same order
refcount_copying(&dst, dtype_is_object, ndim, False)
memcpy(dst.data, src.data, slice_get_size(&src, ndim))
refcount_copying(&dst, dtype_is_object, ndim, True)
return 0
if order == 'F' == get_best_order(&dst, ndim):
......@@ -1185,7 +1204,10 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src,
transpose_memslice(&src)
transpose_memslice(&dst)
refcount_copying(&dst, dtype_is_object, ndim, False)
copy_strided_to_strided(&src, &dst, ndim, itemsize)
refcount_copying(&dst, dtype_is_object, ndim, True)
free(tmpdata)
return 0
......@@ -1206,6 +1228,38 @@ cdef void broadcast_leading({{memviewslice_name}} *slice,
slice.strides[i] = slice.strides[0]
slice.suboffsets[i] = -1
@cname('__pyx_memoryview_refcount_copying')
cdef void refcount_copying({{memviewslice_name}} *dst, bint dtype_is_object,
int ndim, bint inc) nogil:
# incref or decref the objects in the destination slice if the dtype is
# object
if dtype_is_object:
refcount_objects_in_slice_with_gil(dst.data, dst.shape,
dst.strides, ndim, inc)
@cname('__pyx_memoryview_refcount_objects_in_slice_with_gil')
cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape,
Py_ssize_t *strides, int ndim,
bint inc) with gil:
refcount_objects_in_slice(data, shape, strides, ndim, inc)
@cname('__pyx_memoryview_refcount_objects_in_slice')
cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape,
Py_ssize_t *strides, int ndim, bint inc):
cdef Py_ssize_t i
for i in range(shape[0]):
if ndim == 1:
if inc:
Py_INCREF((<PyObject **> data)[0])
else:
Py_DECREF((<PyObject **> data)[0])
else:
refcount_objects_in_slice(data, shape + 1, strides + 1,
ndim - 1, inc)
data += strides[0]
############### BufferFormatFromTypeInfo ###############
cdef extern from *:
......
Cython/Utility/MemoryView_C.c
View file @ 3693a7b8
......@@ -162,7 +162,7 @@ static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
{{memviewslice_name}} result = {{memslice_init}};
struct __pyx_memoryview_obj *memview = \
(struct __pyx_memoryview_obj *) __pyx_memoryview_new(obj, {{buf_flag}});
(struct __pyx_memoryview_obj *) __pyx_memoryview_new(obj, {{buf_flag}}, 0);
__Pyx_BufFmt_StackElem stack[{{struct_nesting_depth}}];
int axes_specs[] = { {{axes_specs}} };
int retcode;
......@@ -464,13 +464,15 @@ static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
static {{memviewslice_name}}
__pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs,
const char *mode, int ndim,
size_t sizeof_dtype, int contig_flag);
size_t sizeof_dtype, int contig_flag,
int dtype_is_object);
////////// MemviewSliceCopyTemplate //////////
static {{memviewslice_name}}
__pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs,
const char *mode, int ndim,
size_t sizeof_dtype, int contig_flag)
size_t sizeof_dtype, int contig_flag,
int dtype_is_object)
{
__Pyx_RefNannyDeclarations
int i;
......@@ -515,7 +517,8 @@ __pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs,
__Pyx_GOTREF(array_obj);
memview_obj = (struct __pyx_memoryview_obj *) __pyx_memoryview_new(
(PyObject *) array_obj, contig_flag);
(PyObject *) array_obj, contig_flag,
dtype_is_object);
if (unlikely(!memview_obj))
goto fail;
......@@ -523,7 +526,8 @@ __pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs,
if (unlikely(__Pyx_init_memviewslice(memview_obj, ndim, &new_mvs) < 0))
goto fail;
if (unlikely(__pyx_memoryview_copy_contents(*from_mvs, new_mvs, ndim, ndim) < 0))
if (unlikely(__pyx_memoryview_copy_contents(*from_mvs, new_mvs, ndim, ndim,
dtype_is_object) < 0))
goto fail;
goto no_fail;
......@@ -544,7 +548,8 @@ no_fail:
////////// CopyContentsUtility.proto /////////
#define {{func_cname}}(slice) \
__pyx_memoryview_copy_new_contig(&slice, "{{mode}}", {{ndim}}, \
sizeof({{dtype_decl}}), {{contig_flag}})
sizeof({{dtype_decl}}), {{contig_flag}}, \
{{dtype_is_object}})
////////// OverlappingSlices.proto //////////
static int __pyx_slices_overlap({{memviewslice_name}} *slice1,
......
tests/run/memslice.pyx
View file @ 3693a7b8
......@@ -1880,3 +1880,64 @@ cdef _not_borrowed2(int[:] m):
print m[5]
if object():
m = carray
class SingleObject(object):
def __init__(self, value):
self.value = value
def __str__(self):
return str(self.value)
@testcase
def test_object_dtype_copying():
"""
>>> test_object_dtype_copying()
True
0
1
2
3
4
5
6
7
8
9
3 5
2 5
"""
cdef int i
none_refcount = get_refcount(None)
cdef cython.array a1 = cython.array((10,), sizeof(PyObject *), 'O')
cdef cython.array a2 = cython.array((10,), sizeof(PyObject *), 'O')
print a1.dtype_is_object
cdef object[:] m1 = a1
cdef object[:] m2 = a2
for i in range(10):
# Initialize to None first
(<PyObject **> a1.data)[i] = <PyObject *> None
Py_INCREF(None)
(<PyObject **> a2.data)[i] = <PyObject *> None
Py_INCREF(None)
# now set a unique object
m1[i] = SingleObject(i)
m2[...] = m1
del a1, a2, m1
for i in range(10):
print m2[i]
obj = m2[5]
print get_refcount(obj), obj
del m2
print get_refcount(obj), obj
assert none_refcount == get_refcount(None)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7