Skip to content

C
cython
Commit b7e14b9a authored by Mark Florisson's avatar Mark Florisson
Browse files
Options

cimport cython in CyUtility, more tests, nogil slicing

parent 8105941c
Hide whitespace changes
Inline Side-by-side
Showing with 561 additions and 340 deletions
Cython/Compiler/Code.pxd
View file @ b7e14b9a
......@@ -13,6 +13,8 @@ cdef class UtilityCode(UtilityCodeBase):
cdef public dict _cache
cdef public list specialize_list
cdef public object proto_block
cdef public object name
cdef public object file
cpdef put_code(self, output)
......
Cython/Compiler/CythonScope.py
View file @ b7e14b9a
......@@ -75,10 +75,13 @@ class CythonScope(ModuleScope):
Creates some entries for testing purposes and entries for
cython.array() and for cython.view.*.
"""
cython_testscope_utility_code.declare_in_scope(self)
cython_test_extclass_utility_code.declare_in_scope(self)
cython_testscope_utility_code.declare_in_scope(
self, cython_scope=self)
cython_test_extclass_utility_code.declare_in_scope(
self, cython_scope=self)
MemoryView.cython_array_utility_code.declare_in_scope(self)
MemoryView.cython_array_utility_code.declare_in_scope(
self, cython_scope=self)
#
# The view sub-scope
......@@ -88,9 +91,11 @@ class CythonScope(ModuleScope):
viewscope.is_cython_builtin = True
viewscope.pxd_file_loaded = True
cythonview_testscope_utility_code.declare_in_scope(viewscope)
cythonview_testscope_utility_code.declare_in_scope(
viewscope, cython_scope=self)
view_utility_scope = MemoryView.view_utility_code.declare_in_scope(viewscope)
view_utility_scope = MemoryView.view_utility_code.declare_in_scope(
viewscope, cython_scope=self)
# MemoryView.memview_fromslice_utility_code.from_scope = view_utility_scope
# MemoryView.memview_fromslice_utility_code.declare_in_scope(viewscope)
......
Cython/Compiler/ExprNodes.py
View file @ b7e14b9a
......@@ -2432,7 +2432,8 @@ class IndexNode(ExprNode):
import MemoryView
skip_child_analysis = True
indices = MemoryView.unellipsify(indices, self.base.type.ndim)
have_slices, indices = MemoryView.unellipsify(indices,
self.base.type.ndim)
self.memslice_index = len(indices) == self.base.type.ndim
axes = []
......@@ -2462,24 +2463,19 @@ class IndexNode(ExprNode):
value = getattr(index, attr)
if not value.is_none:
value = value.coerce_to(index_type, env)
value = value.coerce_to_temp(env)
#value = value.coerce_to_temp(env)
setattr(index, attr, value)
new_indices.append(value)
elif index.type.is_int:
self.memslice_index = True
index = index.coerce_to(index_type, env).coerce_to_temp(
index_type)
index = index.coerce_to(index_type, env)\
#.coerce_to_temp(
# index_type)
indices[i] = index
new_indices.append(index)
if access in ('ptr', 'generic') and i != 0:
# If this dimension is to disappear, then how do we
# indicate that we need to dereference in this dimension
# if the previous dimension is already indirect, or if
# the previous dimension was direct but also indexed?
# Basically only a[i, j, k, :] can work, as you can
# set the base pointer to start in the fourth dimension
if access in ('ptr', 'generic') and i != 0 and have_slices:
self.type = error_type
return error(index.pos,
"Indexing of non-leading indirect or generic "
......@@ -2494,6 +2490,8 @@ class IndexNode(ExprNode):
self.original_indices = indices
self.indices = new_indices
self.env = env
elif self.base.type.is_buffer:
# Buffer indexing
if len(indices) == self.base.type.ndim:
......@@ -2893,7 +2891,8 @@ class IndexNode(ExprNode):
self.original_indices,
self.base.type,
self.type,
self.result())
self.result(),
have_gil = not self.env.nogil)
def put_nonecheck(self, code):
code.globalstate.use_utility_code(raise_noneindex_error_utility_code)
......@@ -4092,8 +4091,9 @@ class AttributeNode(ExprNode):
def nogil_check(self, env):
if self.is_py_attr:
self.gil_error()
import MemoryView
MemoryView.err_if_nogil_initialized_check(self.pos, env, 'attribute')
elif self.type.is_memoryviewslice:
import MemoryView
MemoryView.err_if_nogil_initialized_check(self.pos, env, 'attribute')
gil_message = "Accessing Python attribute"
......
Cython/Compiler/MemoryView.py
View file @ b7e14b9a
......@@ -228,189 +228,72 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
return bufp
def generate_buffer_slice_code(self, code, indices, type, dst_type, dst):
"""
Generate code for a new memoryview slice.
indices An index in the indices list is either a SliceNode where
start/stop and step are converted to temps of type Py_ssize_t, or a
temp integer index of type Py_ssize_t.
type is the type that is sliced
dst_type is the resulting type
dst is the result memoryview slice temp
For every dimension we do the following:
ensure 0 <= start < n and 0 <= stop <= n
if every dimension is direct:
keep track of possible offsets, set data * to
mslice[0, 0, 0, ...] after the loop
else:
keep a variable 'dim' that
if < 0
indicates the data *
else
indicates suboffsets[dim]
now set shape, strides and suboffsets according to start/stop/step
add the slice or indexing offset to data * or to suboffsets[dim]
update the dim variable if necessary
"""
axes = []
temps = []
dtype = PyrexTypes.c_py_ssize_t_type
def put_bounds_code(r, start):
"ensure that start 0 <= r < n"
code.putln("if (%s < 0) {" % r)
code.putln( "%s += %s;" % (r, shape))
code.putln( "if (%s < 0) %s = 0;" % (r, r))
if start:
code.putln("} else if (%s >= %s) %s = %s - 1;" % (r, shape, r, shape))
def generate_buffer_slice_code(self, code, indices, type, dst_type, dst, have_gil):
slicefunc = "__pyx_memoryview_slice_memviewslice"
new_ndim = 0
cname = self.cname
suboffset_dim = code.funcstate.allocate_temp(PyrexTypes.c_int_type,
False)
index_code = ("%(slicefunc)s(&%(cname)s, &%(dst)s, %(have_gil)d, "
"%(dim)d, %(new_ndim)d, &%(suboffset_dim)s, "
"%(idx)s, 0, 0, 0, 0, 0, 0)")
slice_code = ("%(slicefunc)s(&%(cname)s, &%(dst)s, %(have_gil)d, "
"/* dim */ %(dim)d, "
"/* new_ndim */ %(new_ndim)d, "
"/* suboffset_dim */ &%(suboffset_dim)s, "
"/* start */ %(start)s, "
"/* stop */ %(stop)s, "
"/* step */ %(step)s, "
"/* have_start */ %(have_start)d, "
"/* have_stop */ %(have_stop)d, "
"/* have_step */ %(have_step)d, "
"/* is_slice */ 1)")
def generate_slice_call(expr):
pos = index.pos
if have_gil:
code.putln(code.error_goto_if(expr, pos))
else:
code.putln("} else if (%s > %s) %s = %s;" % (r, shape, r, shape))
def update_dim():
"if we are indirect, update our dim index"
if access in ('ptr', 'generic'):
if access == 'generic':
code.put("if (%s >= 0)" % suboffset)
code.putln("%s = %d;" % (offset_dim, dst_dim))
all_direct = True
for access, packing in type.axes:
all_direct = all_direct and access == 'direct'
if not all_direct:
break
if not all_direct:
offset_dim = code.funcstate.allocate_temp(PyrexTypes.c_int_type,
False)
code.putln("%s = -1;" % (offset_dim,))
code.putln("%s.data = %s.data;" % (dst, self.cname))
dst_dim = 0
for dim, index in enumerate(indices):
goto_err = code.error_goto(index.pos)
shape = "%s.shape[%d]" % (self.cname, dim)
stride = "%s.strides[%d]" % (self.cname, dim)
suboffset = "%s.suboffsets[%d]" % (self.cname, dim)
dst_shape = "%s.shape[%d]" % (dst, dst_dim)
dst_stride = "%s.strides[%d]" % (dst, dst_dim)
dst_suboffset = "%s.suboffsets[%d]" % (dst, dst_dim)
access, packing = type.axes[dim]
code.putln("{")
code.putln( "const char *__pyx_t_result = %s;" % expr)
if isinstance(index, ExprNodes.SliceNode):
# slice or part of ellipsis
code.putln( "if (unlikely(__pyx_t_result)) {")
code.put_ensure_gil()
code.putln( "PyErr_Format(PyExc_IndexError, "
"__pyx_t_result, %d)" % dim)
code.put_release_ensured_gil()
code.putln(code.goto_error(pos))
code.putln( "}")
start = stop = step = None
dst_dim += 1
# First fix the bounds
if not index.start.is_none:
start = index.start.result()
put_bounds_code(start, start=True)
if not index.stop.is_none:
stop = index.stop.result()
put_bounds_code(stop, start=False)
# Compute the new strides
if not index.step.is_none:
step = index.step.result()
code.putln("%s = %s * %s;" % (dst_stride, stride, step))
else:
code.putln("%s = %s;" % (dst_stride, stride))
# Take care of suboffsets
code.putln("%s = %s;" % (dst_suboffset, suboffset))
# If start or stop is not specified, then we need to set
# them according to step
if not start or not stop:
if step:
code.putln("if (%s > 0) {" % step)
if not start:
start = code.funcstate.allocate_temp(dtype, False)
temps.append(start)
code.putln("%s = 0;" % start)
if not stop:
stop = code.funcstate.allocate_temp(dtype, False)
temps.append(stop)
code.putln("%s = %s;" % (stop, shape))
code.putln("} else {")
if start:
code.putln("%s = %s - 1;" % (start, shape))
if stop:
code.putln("%s = -1;" % stop)
code.putln("}")
else:
if not start:
start = "0"
if not stop:
stop = shape
d = dict(locals(), step=step or "1")
# Take care of shape
code.putln("%(dst_shape)s = (%(stop)s - %(start)s) / %(step)s;" % d)
code.putln("if (%(dst_shape)s && (%(stop)s - %(start)s) %% %(step)s) %(dst_shape)s++;" % d)
code.putln("if (%(dst_shape)s < 0) %(dst_shape)s = 0;" % d)
# Take care of slicing offsets
if start != "0":
if all_direct:
axes.append([dim, start, access, packing])
else:
offset = "%s * %s" % (start, stride)
d = dict(dim=offset_dim, dst=dst, offset=offset)
code.putln("if (%(dim)s < 0) %(dst)s.data += %(offset)s;" % d)
code.putln("else %(dst)s.suboffsets[%(dim)s] += %(offset)s;" % d)
else:
# integer index (or object converted to integer index)
# Note: this dimension disappears
assert access == 'direct'
r = index.result()
# Bounds checking with error
code.globalstate.use_utility_code(Buffer.raise_indexerror_code)
out_of_bounds = "%s < 0 || %s >= %s" % (r, r, shape)
code.putln("if (%s) {" % code.unlikely(out_of_bounds))
code.putln('__Pyx_RaiseBufferIndexError(%d);' % dim)
code.putln(code.error_goto(index.pos))
code.putln("}")
if all_direct:
axes.append([dim, index.result(), access, packing])
else:
add_slice_offset(r)
code.putln("%s = -1;" % suboffset_dim)
code.putln("%(dst)s.data = %(cname)s.data;" % locals())
code.putln("%(dst)s.memview = %(cname)s.memview;" % locals())
# Take care of data * for direct access in all dimensions
if all_direct:
bufp = self._generate_buffer_lookup_code(code, axes,
cast_result=False)
code.putln("%s.data = %s;" % (dst, bufp))
for dim, index in enumerate(indices):
if not isinstance(index, ExprNodes.SliceNode):
idx = index.result()
generate_slice_call(index_code % locals())
else:
d = {}
for s in "start stop step".split():
idx = getattr(index, s)
have_idx = d['have_' + s] = not idx.is_none
if have_idx:
d[s] = idx.result()
else:
d[s] = "0"
# Finally take care of memview
code.putln("%s.memview = %s.memview;" % (dst, self.cname))
d.update(locals())
generate_slice_call(slice_code % d)
new_ndim += 1
for temp in temps:
code.funcstate.release_temp(temp)
code.funcstate.release_temp(suboffset_dim)
def empty_slice(pos):
none = ExprNodes.NoneNode(pos)
......@@ -420,10 +303,13 @@ def empty_slice(pos):
def unellipsify(indices, ndim):
result = []
seen_ellipsis = False
have_slices = False
for index in indices:
if isinstance(index, ExprNodes.EllipsisNode):
have_slices = True
full_slice = empty_slice(index.pos)
if seen_ellipsis:
result.append(full_slice)
else:
......@@ -431,13 +317,15 @@ def unellipsify(indices, ndim):
result.extend([full_slice] * nslices)
seen_ellipsis = True
else:
have_slices = have_slices or isinstance(index, ExprNodes.SliceNode)
result.append(index)
if len(result) < ndim:
have_slices = True
nslices = ndim - len(result)
result.extend([empty_slice(indices[-1].pos)] * nslices)
return result
return have_slices, result
def get_memoryview_flag(access, packing):
if access == 'full' and packing in ('strided', 'follow'):
......@@ -885,10 +773,6 @@ view_constant_to_access_packing = {
'indirect_contiguous': ('ptr', 'contig'),
}
def get_access_packing(view_scope_constant):
if view_scope_constant.name == 'generic':
return 'full',
def validate_axes_specs(positions, specs):
packing_specs = ('contig', 'strided', 'follow')
......
Cython/Compiler/UtilityCode.py
View file @ b7e14b9a
......@@ -9,6 +9,7 @@ class NonManglingModuleScope(Symtab.ModuleScope):
def __init__(self, prefix, *args, **kw):
self.prefix = prefix
self.cython_scope = None
Symtab.ModuleScope.__init__(self, *args, **kw)
def add_imported_entry(self, name, entry, pos):
......@@ -31,9 +32,12 @@ class CythonUtilityCodeContext(StringParseContext):
def find_module(self, module_name, relative_to = None, pos = None,
need_pxd = 1):
if module_name != self.module_name:
raise AssertionError("Not yet supporting any cimports/includes "
"from string code snippets")
if module_name not in self.modules:
raise AssertionError("Only the cython cimport is supported.")
else:
return self.modules[module_name]
if self.scope is None:
self.scope = NonManglingModuleScope(self.prefix,
......@@ -78,7 +82,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
self.requires = requires or []
self.from_scope = from_scope
def get_tree(self, entries_only=False):
def get_tree(self, entries_only=False, cython_scope=None):
from AnalysedTreeTransforms import AutoTestDictTransform
# The AutoTestDictTransform creates the statement "__test__ = {}",
# which when copied into the main ModuleNode overwrites
......@@ -88,6 +92,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
import Pipeline, ParseTreeTransforms
context = CythonUtilityCodeContext(self.name)
context.prefix = self.prefix
context.cython_scope = cython_scope
#context = StringParseContext(self.name)
tree = parse_from_strings(self.name, self.impl, context=context,
allow_struct_enum_decorator=True)
......@@ -125,13 +130,13 @@ class CythonUtilityCode(Code.UtilityCodeBase):
def put_code(self, output):
pass
def declare_in_scope(self, dest_scope, used=False):
def declare_in_scope(self, dest_scope, used=False, cython_scope=None):
"""
Declare all entries from the utility code in dest_scope. Code will only
be included for used entries. If module_name is given, declare the
type entries with that name.
"""
tree = self.get_tree(entries_only=True)
tree = self.get_tree(entries_only=True, cython_scope=cython_scope)
entries = tree.scope.entries
entries.pop('__name__')
......
Cython/Utility/MemoryView.pyx
View file @ b7e14b9a
......@@ -165,6 +165,8 @@ cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *
########## View.MemoryView ##########
import cython
# from cpython cimport ...
cdef extern from "Python.h":
int PyIndex_Check(object)
......@@ -245,7 +247,6 @@ cdef class memoryview(object):
cdef char *itemp = <char *> self.view.buf
for dim, idx in enumerate(index):
_check_index(idx)
itemp = pybuffer_index(&self.view, itemp, idx, dim)
return itemp
......@@ -255,13 +256,13 @@ cdef class memoryview(object):
if index is Ellipsis:
return self
have_slices, index = _unellipsify(index, self.view.ndim)
have_slices, indices = _unellipsify(index, self.view.ndim)
cdef char *itemp
if have_slices:
return pybuffer_slice(self, index)
return memview_slice(self, indices)
else:
itemp = self.get_item_pointer(index)
itemp = self.get_item_pointer(indices)
return self.convert_item_to_object(itemp)
@cname('__pyx_memoryview_setitem')
......@@ -355,12 +356,11 @@ cdef class memoryview(object):
cdef memoryview_cwrapper(object o, int flags):
return memoryview(o, flags)
cdef _check_index(index):
if not PyIndex_Check(index):
raise TypeError("Cannot index with %s" % type(index))
cdef tuple _unellipsify(object index, int ndim):
"""
Replace all ellipses with full slices and fill incomplete indices with
full slices.
"""
if not isinstance(index, tuple):
tup = (index,)
else:
......@@ -373,11 +373,14 @@ cdef tuple _unellipsify(object index, int ndim):
if item is Ellipsis:
if not seen_ellipsis:
result.extend([slice(None)] * (ndim - len(tup) + 1))
result.extend(tup[idx + 1:])
seen_ellipsis = True
else:
result.append(slice(None))
have_slices = True
else:
if not isinstance(item, slice) and not PyIndex_Check(item):
raise TypeError("Cannot index with type '%s'" % type(item))
have_slices = have_slices or isinstance(item, slice)
result.append(item)
......@@ -385,102 +388,212 @@ cdef tuple _unellipsify(object index, int ndim):
if nslices:
result.extend([slice(None)] * nslices)
for idx in tup:
if isinstance(idx, slice):
return True, tup
return have_slices or nslices, tuple(result)
return False, tup
#
### Slicing a memoryview
#
@cname('__pyx_pybuffer_slice')
cdef memoryview pybuffer_slice(memoryview memview, object indices):
cdef Py_ssize_t idx, dim, new_dim = 0, suboffset_dim = -1
cdef Py_ssize_t shape, stride
@cname('__pyx_memview_slice')
cdef memoryview memview_slice(memoryview memview, object indices):
cdef int new_ndim = 0, suboffset_dim = -1, dim
cdef bint negative_step
cdef int new_ndim = 0
cdef {{memviewslice_name}} dst
cdef {{memviewslice_name}} dst, src
cdef {{memviewslice_name}} *p_src
for dim, index in enumerate(indices):
shape = memview.view.shape[dim]
stride = memview.view.strides[dim]
cdef _memoryviewslice memviewsliceobj
assert memview.view.ndim > 0
if isinstance(memview, _memoryviewslice):
memviewsliceobj = memview
p_src = &memviewsliceobj.from_slice
else:
create_slice(memview, &src)
p_src = &src
# Note: don't use variable src at this point
# SubNote: we should be able to declare variables in blocks...
# memoryview_fromslice() will inc our dst slice
dst.memview = p_src.memview
dst.data = p_src.data
for dim, index in enumerate(indices):
if PyIndex_Check(index):
idx = index
if idx < 0:
idx += shape
if not 0 <= idx < shape:
raise IndexError("Index out of bounds (axis %d)" % dim)
slice_memviewslice(p_src, &dst, True, dim, new_ndim, &suboffset_dim,
index, 0, 0, 0, 0, 0, False)
else:
# index is a slice
slice_memviewslice(p_src, &dst, True, dim, new_ndim, &suboffset_dim,
index.start or 0,
index.stop or 0,
index.step or 0,
index.start is not None,
index.stop is not None,
index.step is not None,
True)
new_ndim += 1
start, stop, step = index.start, index.stop, index.step
negative_step = step and step < 0
if isinstance(memview, _memoryviewslice):
return memoryview_fromslice(&dst, new_ndim,
memviewsliceobj.to_object_func,
memviewsliceobj.to_dtype_func)
else:
return memoryview_fromslice(&dst, new_ndim, NULL, NULL)
# set some defaults
if not start:
if negative_step:
start = shape - 1
else:
start = 0
#
### Slicing in a single dimension of a memoryviewslice
#
if not stop:
if negative_step:
stop = -1
else:
stop = shape
cdef extern from "stdlib.h":
void abort() nogil
cdef extern from "stdio.h":
ctypedef struct FILE
FILE *stderr
int fputs(char *s, FILE *stream)
cdef extern from "pystate.h":
void PyThreadState_Get() nogil
# These are not actually nogil, but we check for the GIL before calling them
void PyErr_SetString(PyObject *type, char *msg) nogil
PyObject *PyErr_Format(PyObject *exc, char *msg, ...) nogil
cdef:
char *ERR_OOB = "Index out of bounds (axis %d)"
char *ERR_STEP = "Step must not be zero (axis %d)"
char *ERR_INDIRECT_GIL = ("Cannot make indirect dimension %d disappear "
"through indexing, consider slicing with %d:%d")
char *ERR_INDIRECT_NOGIL = ("Cannot make indirect dimension %d disappear "
"through indexing")
PyObject *exc = <PyObject *> IndexError
@cname('__pyx_memoryview_slice_memviewslice')
cdef char *slice_memviewslice({{memviewslice_name}} *src,
{{memviewslice_name}} *dst,
bint have_gil,
int dim,
int new_ndim,
int *suboffset_dim,
Py_ssize_t start,
Py_ssize_t stop,
Py_ssize_t step,
int have_start,
int have_stop,
int have_step,
bint is_slice) nogil except *:
"""
Create a new slice dst given slice src.
have_gil - if true, the GIL must be held and exceptions may be raised
dim - the current src dimension (indexing will make dimensions
disappear)
new_dim - the new dst dimension
suboffset_dim - pointer to a single int initialized to -1 to keep track of
where slicing offsets should be added
"""
# check our bounds
cdef:
Py_ssize_t shape, stride, suboffset
Py_ssize_t new_shape
bint negative_step
if have_gil:
# Assert the GIL
PyThreadState_Get()
shape = src.shape[dim]
stride = src.strides[dim]
suboffset = src.suboffsets[dim]
if not is_slice:
# index is a normal integer-like index
if start < 0:
start += shape
if not 0 <= start < shape:
if have_gil:
PyErr_Format(exc, ERR_OOB, dim)
return ERR_OOB
else:
# index is a slice
negative_step = have_step != 0 and step < 0
if have_step and step == 0:
if have_gil:
# ValueError might be more appropriate, but this will make it consistent
# with nogil slicing
PyErr_SetString(exc, ERR_STEP)
return ERR_STEP
# check our bounds and set defaults
if have_start:
if start < 0:
start += shape
if start < 0:
start = 0
elif start >= shape:
if negative_step:
start = shape - 1
else:
start = shape
else:
if negative_step:
start = shape - 1
else:
start = 0
if have_stop:
if stop < 0:
stop += shape
if stop < 0:
stop = 0
elif stop > shape:
stop = shape
else:
if negative_step:
stop = -1
else:
stop = shape
step = step or 1
if not have_step:
step = 1
# shape/strides/suboffsets
dst.strides[new_dim] = stride * step
dst.shape[new_dim] = (stop - start) / step
if (stop - start) % step:
dst.shape[new_dim] += 1
dst.suboffsets[new_dim] = memview.view.suboffsets[dim]
# len = ceil( (stop - start) / step )
with cython.cdivision(True):
new_shape = (stop - start) // step
# set this for the slicing offset
if negative_step:
idx = stop
else:
idx = start
if (stop - start) % step:
new_shape += 1
# Add the slicing or idexing offsets to the right suboffset or base data *
if suboffset_dim < 0:
dst.data += idx * stride
else:
dst.suboffsets[suboffset_dim] += idx * stride
if new_shape < 0:
new_shape = 0
if memview.view.suboffsets[dim]:
if PyIndex_Check(index):
raise IndexError(
"Cannot make indirect dimension %d disappear through "
"indexing, consider slicing with %d:%d" % (dim, idx, idx + 1))
suboffset_dim = new_dim
# shape/strides/suboffsets
dst.strides[new_ndim] = stride * step
dst.shape[new_ndim] = new_shape
dst.suboffsets[new_ndim] = suboffset
cdef _memoryviewslice memviewsliceobj
if isinstance(memview, _memoryviewslice):
memviewsliceobj = memview
return memoryview_fromslice(&dst, new_dim,
memviewsliceobj.to_object_func,
memviewsliceobj.to_dtype_func)
# Add the slicing or idexing offsets to the right suboffset or base data *
if suboffset_dim[0] < 0:
dst.data += start * stride
else:
return memoryview_fromslice(&dst, new_dim, NULL, NULL)
dst.suboffsets[suboffset_dim[0]] += start * stride
if suboffset >= 0:
if not is_slice:
if have_gil:
PyErr_Format(exc, ERR_INDIRECT_GIL, dim, start, start + 1)
return ERR_INDIRECT_NOGIL
else:
suboffset_dim[0] = new_ndim
return NULL
#
### Index a memoryview
#
@cname('__pyx_pybuffer_index')
cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index,
int dim) except NULL:
......@@ -511,13 +624,16 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index,
return resultp
#
### Creating new memoryview objects from slices and memoryviews
#
@cname('__pyx_memoryviewslice')
cdef class _memoryviewslice(memoryview):
"Internal class for passing memory view slices to Python"
"Internal class for passing memoryview slices to Python"
# We need this to keep our shape/strides/suboffset pointers valid
cdef {{memviewslice_name}} from_slice
# We need this only to print it's classes name
# We need this only to print it's class' name
cdef object from_object
cdef object (*to_object_func)(char *)
......@@ -550,9 +666,12 @@ cdef memoryview_fromslice({{memviewslice_name}} *memviewslice,
object (*to_object_func)(char *),
int (*to_dtype_func)(char *, object) except 0):
cdef _memoryviewslice result
cdef int i
assert 0 < ndim <= memviewslice.memview.view.ndim, (ndim, memviewslice.memview.view.ndim)
cdef _memoryviewslice result = _memoryviewslice(None, 0)
result = _memoryviewslice(None, 0)
result.from_slice = memviewslice[0]
__PYX_INC_MEMVIEW(memviewslice, 1)
......@@ -560,30 +679,41 @@ cdef memoryview_fromslice({{memviewslice_name}} *memviewslice,
result.from_object = <object> memviewslice.memview.obj
result.view = memviewslice.memview.view
result.view.shape = <Py_ssize_t *> &result.from_slice.shape
result.view.strides = <Py_ssize_t *> result.from_slice.strides
result.view.suboffsets = <Py_ssize_t *> &result.from_slice.suboffsets
result.view.buf = <void *> memviewslice.data
result.view.ndim = ndim
result.view.shape = <Py_ssize_t *> result.from_slice.shape
result.view.strides = <Py_ssize_t *> result.from_slice.strides
result.view.suboffsets = <Py_ssize_t *> result.from_slice.suboffsets
result.view.len = result.view.itemsize
for i in range(ndim):
result.view.len *= result.view.shape[i]
result.to_object_func = to_object_func
result.to_dtype_func = to_dtype_func
return result
@cname('__pyx_memoryview_create_slice')
cdef void create_slice(memoryview memview, {{memviewslice_name}} *dst):
cdef int dim
dst.memview = <__pyx_memoryview *> memview
dst.data = <char *> memview.view.buf
for dim in range(memview.view.ndim):
dst.shape[dim] = memview.view.shape[dim]
dst.strides[dim] = memview.view.strides[dim]
dst.suboffsets[dim] = memview.view.suboffsets[dim]
@cname('__pyx_memoryview_copy')
cdef memoryview_copy(memoryview memview):
cdef {{memviewslice_name}} memviewslice
cdef int dim
cdef object (*to_object_func)(char *)
cdef int (*to_dtype_func)(char *, object) except 0
memviewslice.memview = <__pyx_memoryview *> memview
memviewslice.data = <char *> memview.view.buf
# Copy all of these as from_slice will
for dim in range(memview.view.ndim):
memviewslice.shape[dim] = memview.view.shape[dim]
memviewslice.strides[dim] = memview.view.strides[dim]
memviewslice.suboffsets[dim] = memview.view.suboffsets[dim]
create_slice(memview, &memviewslice)
if isinstance(memview, _memoryviewslice):
to_object_func = (<_memoryviewslice> memview).to_object_func
......
Cython/Utility/MemoryView_C.c
View file @ b7e14b9a
......@@ -284,7 +284,7 @@ static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
if (!memview)
return; /* allow uninitialized memoryview assignment */
if (memview->acquisition_count <= 0)
if (memview->acquisition_count < 0)
__pyx_fatalerror("Acquisition count is %d (line %d)",
memview->acquisition_count, lineno);
......@@ -463,4 +463,4 @@ int {{set_function}}(const char *itemp, PyObject *obj) {
Py_DECREF(*(PyObject **) itemp);
*(PyObject **) itemp = obj;
return 1;
}
\ No newline at end of file
}
tests/run/memoryview.pyx
View file @ b7e14b9a
......@@ -588,40 +588,122 @@ def assign_temporary_to_object(object[:] mslice):
buf = mslice
buf[1] = {3-2: 2+(2*4)-2}
def test_slicing(arg):
def print_int_offsets(*args):
for item in args:
print item / sizeof(int),
print
def test_generic_slicing(arg):
"""
Test simple slicing
>>> test_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
>>> test_generic_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
acquired A
3 9 2
1232 -44 4
(3, 9, 2)
308 -11 1
-1 -1 -1
released A
Test direct slicing, negative slice oob in dim 2
>>> test_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
>>> test_generic_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
acquired A
0 0 2
48 -12 4
(0, 0, 2)
12 -3 1
-1 -1 -1
released A
Test indirect slicing
>>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
>>> test_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
>>> test_generic_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
acquired A
2 0 2
8 -4 4
0 0 -1
(2, 0, 2)
0 1 -1
released A
>>> stride1 = 21 * 14
>>> stride2 = 21
>>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
>>> test_generic_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
acquired A
(3, 9, 2)
20 1 -1
released A
"""
cdef int[::view.generic, ::view.generic, :] _a = arg
a = _a
b = a[2:8:2, -4:1:-1, 1:3]
print b.shape[0], b.shape[1], b.shape[2]
print b.strides[0], b.strides[1], b.strides[2]
print b.suboffsets[0], b.suboffsets[1], b.suboffsets[2]
print b.shape
if b.suboffsets[0] < 0:
print_int_offsets(*b.strides)
print_int_offsets(*b.suboffsets)
cdef int i, j, k
for i in range(b.shape[0]):
for j in range(b.shape[1]):
for k in range(b.shape[2]):
itemA = a[2 + 2 * i, -4 - j, 1 + k]
itemB = b[i, j, k]
assert itemA == itemB, (i, j, k, itemA, itemB)
def test_indirect_slicing(arg):
"""
Test indirect slicing
>>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
>>> test_indirect_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
acquired A
(5, 3, 2)
0 0 -1
58
released A
>>> stride1 = 21 * 14
>>> stride2 = 21
>>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
>>> test_indirect_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
acquired A
(5, 14, 3)
0 16 -1
2412
released A
"""
cdef int[::view.indirect, ::view.indirect, :] _a = arg
a = _a
b = a[-5:, ..., -5:100:2]
print b.shape
print_int_offsets(*b.suboffsets)
print b[4, 2, 1]
def test_direct_slicing(arg):
"""
Fused types would be convenient to test this stuff!
Test simple slicing
>>> test_direct_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
acquired A
(3, 9, 2)
308 -11 1
-1 -1 -1
released A
Test direct slicing, negative slice oob in dim 2
>>> test_direct_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
acquired A
(0, 0, 2)
12 -3 1
-1 -1 -1
released A
"""
cdef int[:, :, :] _a = arg
a = _a
b = a[2:8:2, -4:1:-1, 1:3]
print b.shape
print_int_offsets(*b.strides)
print_int_offsets(*b.suboffsets)
cdef int i, j, k
for i in range(b.shape[0]):
......@@ -631,13 +713,14 @@ def test_slicing(arg):
itemB = b[i, j, k]
assert itemA == itemB, (i, j, k, itemA, itemB)
def test_slicing_and_indexing(arg):
"""
>>> a = IntStridedMockBuffer("A", range(10 * 3 * 5), shape=(10, 3, 5))
>>> test_slicing_and_indexing(a)
acquired A
5 2
60 8
(5, 2)
15 2
126 113
[111]
released A
......@@ -648,8 +731,8 @@ def test_slicing_and_indexing(arg):
c = b[4:1:-1, ::-1]
d = c[2, 1:2]
print b.shape[0], b.shape[1]
print b.strides[0], b.strides[1]
print b.shape
print_int_offsets(*b.strides)
cdef int i, j
for i in range(b.shape[0]):
......@@ -659,4 +742,14 @@ def test_slicing_and_indexing(arg):
assert itemA == itemB, (i, j, itemA, itemB)
print c[1, 1], c[2, 0]
print [d[i] for i in range(d.shape[0])]
\ No newline at end of file
print [d[i] for i in range(d.shape[0])]
def test_oob():
"""
>>> test_oob()
Traceback (most recent call last):
...
IndexError: Index out of bounds (axis 1)
"""
cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9))
print a[:, 20]
tests/run/memslice.pyx
View file @ b7e14b9a
......@@ -2,6 +2,7 @@
from __future__ import unicode_literals
cimport cython
from cython cimport view
__test__ = {}
......@@ -1158,40 +1159,127 @@ def test_cdef_function2():
cdef_function2(global_A, global_B)
def print_int_offsets(*args):
for item in args:
print item / sizeof(int),
print
@testcase
def test_slicing(arg):
def test_generic_slicing(arg):
"""
Test simple slicing
>>> test_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
>>> test_generic_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
acquired A
3 9 2
1232 -44 4
308 -11 1
-1 -1 -1
released A
Test direct slicing, negative slice oob in dim 2
>>> test_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
>>> test_generic_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
acquired A
0 0 2
48 -12 4
12 -3 1
-1 -1 -1
released A
Test indirect slicing
>>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
>>> test_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
>>> test_generic_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
acquired A
2 0 2
8 -4 4
0 0 -1
0 1 -1
released A
>>> stride1 = 21 * 14
>>> stride2 = 21
>>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
>>> test_generic_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
acquired A
3 9 2
20 1 -1
released A
"""
cdef int[::view.generic, ::view.generic, :] a = arg
cdef int[::view.generic, ::view.generic, :] b = a[2:8:2, -4:1:-1, 1:3]
print b.shape[0], b.shape[1], b.shape[2]
print b.strides[0], b.strides[1], b.strides[2]
print b.suboffsets[0], b.suboffsets[1], b.suboffsets[2]
if b.suboffsets[0] < 0:
print_int_offsets(b.strides[0], b.strides[1], b.strides[2])
print_int_offsets(b.suboffsets[0], b.suboffsets[1], b.suboffsets[2])
cdef int i, j, k
for i in range(b.shape[0]):
for j in range(b.shape[1]):
for k in range(b.shape[2]):
itemA = a[2 + 2 * i, -4 - j, 1 + k]
itemB = b[i, j, k]
assert itemA == itemB, (i, j, k, itemA, itemB)
@testcase
def test_indirect_slicing(arg):
"""
Test indirect slicing
>>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
>>> test_indirect_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
acquired A
5 3 2
0 0 -1
58
56
released A
>>> stride1 = 21 * 14
>>> stride2 = 21
>>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
>>> test_indirect_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
acquired A
5 14 3
0 16 -1
2412
2410
released A
"""
cdef int[::view.indirect, ::view.indirect, :] a = arg
cdef int[::view.indirect, ::view.indirect, :] b = a[-5:, ..., -5:100:2]
cdef int[::view.indirect, ::view.indirect] c = b[..., 0]
print b.shape[0], b.shape[1], b.shape[2]
print_int_offsets(b.suboffsets[0], b.suboffsets[1], b.suboffsets[2])
print b[4, 2, 1]
print c[4, 2]
@testcase
def test_direct_slicing(arg):
"""
Fused types would be convenient to test this stuff!
Test simple slicing
>>> test_direct_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
acquired A
3 9 2
308 -11 1
-1 -1 -1
released A
Test direct slicing, negative slice oob in dim 2
>>> test_direct_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
acquired A
0 0 2
12 -3 1
-1 -1 -1
released A
"""
cdef int[:, :, ::1] a = arg
cdef int[:, :, :] b = a[2:8:2, -4:1:-1, 1:3]
print b.shape[0], b.shape[1], b.shape[2]
print_int_offsets(b.strides[0], b.strides[1], b.strides[2])
print_int_offsets(b.suboffsets[0], b.suboffsets[1], b.suboffsets[2])
cdef int i, j, k
for i in range(b.shape[0]):
......@@ -1208,7 +1296,7 @@ def test_slicing_and_indexing(arg):
>>> test_slicing_and_indexing(a)
acquired A
5 2
60 8
15 2
126 113
[111]
released A
......@@ -1219,7 +1307,7 @@ def test_slicing_and_indexing(arg):
cdef int[:] d = c[2, 1:2]
print b.shape[0], b.shape[1]
print b.strides[0], b.strides[1]
print_int_offsets(b.strides[0], b.strides[1])
cdef int i, j
for i in range(b.shape[0]):
......@@ -1229,4 +1317,16 @@ def test_slicing_and_indexing(arg):
assert itemA == itemB, (i, j, itemA, itemB)
print c[1, 1], c[2, 0]
print [d[i] for i in range(d.shape[0])]
\ No newline at end of file
print [d[i] for i in range(d.shape[0])]
@testcase
def test_oob():
"""
>>> test_oob()
Traceback (most recent call last):
...
IndexError: Index out of bounds (axis 1)
"""
cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9))
print a[:, 20]
tests/run/numpy_memoryview.pyx
View file @ b7e14b9a
......@@ -6,13 +6,15 @@ Test slicing for memoryviews and memoryviewslices
"""
cimport numpy as np
import numpy
import numpy as np
ctypedef np.int32_t dtype_t
def get_array():
# We need to type our array to get a __pyx_get_buffer() that typechecks
# for np.ndarray and calls __getbuffer__ in numpy.pxd
cdef np.ndarray[int, ndim=3] a
a = numpy.arange(8 * 14 * 11).reshape(8, 14, 11)
cdef np.ndarray[dtype_t, ndim=3] a
a = np.arange(8 * 14 * 11, dtype=np.int32).reshape(8, 14, 11)
return a
a = get_array()
......@@ -31,11 +33,11 @@ def test_partial_slicing(array):
"""
>>> test_partial_slicing(a)
"""
cdef int[:, :, :] a = array
cdef dtype_t[:, :, :] a = array
obj = array[4]
cdef int[:, :] b = a[4, :]
cdef int[:, :] c = a[4]
cdef dtype_t[:, :] b = a[4, :]
cdef dtype_t[:, :] c = a[4]
ae(b.shape[0], c.shape[0], obj.shape[0])
ae(b.shape[1], c.shape[1], obj.shape[1])
......@@ -46,15 +48,15 @@ def test_ellipsis(array):
"""
>>> test_ellipsis(a)
"""
cdef int[:, :, :] a = array
cdef dtype_t[:, :, :] a = array
cdef int[:, :] b = a[..., 4]
cdef dtype_t[:, :] b = a[..., 4]
b_obj = array[..., 4]
cdef int[:, :] c = a[4, ...]
cdef dtype_t[:, :] c = a[4, ...]
c_obj = array[4, ...]
cdef int[:, :] d = a[2:8, ..., 2]
cdef dtype_t[:, :] d = a[2:8, ..., 2]
d_obj = array[2:8, ..., 2]
ae(tuple([b.shape[i] for i in range(2)]), b_obj.shape)
......@@ -75,7 +77,7 @@ def test_ellipsis(array):
for j in range(d.shape[1]):
ae(d[i, j], d_obj[i, j])
cdef int[:] e = a[..., 5, 6]
cdef dtype_t[:] e = a[..., 5, 6]
e_obj = array[..., 5, 6]
ae(e.shape[0], e_obj.shape[0])
ae(e.strides[0], e_obj.strides[0])
......@@ -88,7 +90,7 @@ def test_partial_slicing_memoryview(array):
"""
>>> test_partial_slicing_memoryview(a)
"""
cdef int[:, :, :] _a = array
cdef dtype_t[:, :, :] _a = array
a = _a
obj = array[4]
......@@ -104,7 +106,7 @@ def test_ellipsis_memoryview(array):
"""
>>> test_ellipsis_memoryview(a)
"""
cdef int[:, :, :] _a = array
cdef dtype_t[:, :, :] _a = array
a = _a
b = a[..., 4]
......@@ -137,4 +139,4 @@ def test_ellipsis_memoryview(array):
e = a[..., 5, 6]
e_obj = array[..., 5, 6]
ae(e.shape[0], e_obj.shape[0])
ae(e.strides[0], e_obj.strides[0])
\ No newline at end of file
ae(e.strides[0], e_obj.strides[0])
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7