Skip to content

C
cython
Commit 2e17dee8 authored by Mark Florisson's avatar Mark Florisson
Browse files
Options

Memslice nogil tests, better acquisition counting

parent b7e14b9a
Hide whitespace changes
Inline Side-by-side
Showing with 154 additions and 52 deletions
Cython/Compiler/Code.py
View file @ 2e17dee8
......@@ -462,10 +462,11 @@ class FunctionState(object):
A C string referring to the variable is returned.
"""
if not type.is_pyobject:
if not type.is_pyobject and not type.is_memoryviewslice:
# Make manage_ref canonical, so that manage_ref will always mean
# a decref is needed.
manage_ref = False
freelist = self.temps_free.get((type, manage_ref))
if freelist is not None and len(freelist) > 0:
result = freelist.pop()
......@@ -508,7 +509,7 @@ class FunctionState(object):
for name, type, manage_ref in self.temps_allocated:
freelist = self.temps_free.get((type, manage_ref))
if freelist is None or name not in freelist:
used.append((name, type, manage_ref))
used.append((name, type, manage_ref and type.is_pyobject))
return used
def temps_holding_reference(self):
......@@ -518,14 +519,14 @@ class FunctionState(object):
"""
return [(name, type)
for name, type, manage_ref in self.temps_in_use()
if manage_ref]
if manage_ref and type.is_pyobject]
def all_managed_temps(self):
"""Return a list of (cname, type) tuples of refcount-managed Python objects.
"""
return [(cname, type)
for cname, type, manage_ref in self.temps_allocated
if manage_ref]
for cname, type, manage_ref in self.temps_allocated
if manage_ref]
def all_free_managed_temps(self):
"""Return a list of (cname, type) tuples of refcount-managed Python
......@@ -534,9 +535,9 @@ class FunctionState(object):
error case.
"""
return [(cname, type)
for (type, manage_ref), freelist in self.temps_free.items()
if manage_ref
for cname in freelist]
for (type, manage_ref), freelist in self.temps_free.items()
if manage_ref
for cname in freelist]
def start_collecting_temps(self):
"""
......@@ -1464,6 +1465,8 @@ class CCodeWriter(object):
decl = type.declaration_code(name)
if type.is_pyobject:
self.putln("%s = NULL;" % decl)
elif type.is_memoryviewslice:
self.putln("%s = { 0 };" % decl)
else:
self.putln("%s;" % decl)
......@@ -1544,13 +1547,21 @@ class CCodeWriter(object):
self.putln("Py_DECREF(%s); %s = 0;" % (
typecast(py_object_type, type, cname), cname))
def put_xdecref(self, cname, type, nanny=True):
def put_xdecref(self, cname, type, nanny=True, have_gil=True):
if type.is_memoryviewslice:
self.put_xdecref_memoryviewslice(cname, have_gil=have_gil)
return
if nanny:
self.putln("__Pyx_XDECREF(%s);" % self.as_pyobject(cname, type))
else:
self.putln("Py_XDECREF(%s);" % self.as_pyobject(cname, type))
def put_xdecref_clear(self, cname, type, nanny=True):
if type.is_memoryviewslice:
self.put_xdecref_memoryviewslice(cname)
return
if nanny:
self.putln("__Pyx_XDECREF(%s); %s = 0;" % (
self.as_pyobject(cname, type), cname))
......
Cython/Compiler/ExprNodes.py
View file @ 2e17dee8
......@@ -504,8 +504,12 @@ class ExprNode(Node):
def generate_disposal_code(self, code):
if self.is_temp:
if self.type.is_pyobject and self.result():
code.put_decref_clear(self.result(), self.ctype())
if self.result():
if self.type.is_pyobject:
code.put_decref_clear(self.result(), self.ctype())
elif self.type.is_memoryviewslice:
code.put_xdecref_memoryviewslice(
self.result(), have_gil=not self.in_nogil_context)
else:
# Already done if self.is_temp
self.generate_subexpr_disposal_code(code)
......@@ -520,6 +524,9 @@ class ExprNode(Node):
if self.is_temp:
if self.type.is_pyobject:
code.putln("%s = 0;" % self.result())
elif self.type.is_memoryviewslice:
code.putln("%s.memview = NULL;" % self.result())
code.putln("%s.data = NULL;" % self.result())
else:
self.generate_subexpr_disposal_code(code)
......@@ -1477,6 +1484,7 @@ class NameNode(AtomicExprNode):
elif entry.type.is_memoryviewslice:
self.is_temp = False
self.is_used_as_rvalue = True
self.use_managed_ref = True
def nogil_check(self, env):
self.nogil = True
......@@ -1635,6 +1643,11 @@ class NameNode(AtomicExprNode):
raise_unbound = (
(self.cf_maybe_null or self.cf_is_null) and not self.allow_null)
null_code = entry.type.check_for_null_code(entry.cname)
# if entry.type.is_memoryviewslice:
# have_gil = not self.in_nogil_context
# code.put_incref_memoryviewslice(entry.cname, have_gil)
memslice_check = entry.type.is_memoryviewslice and self.initialized_check
if null_code and raise_unbound and (entry.type.is_pyobject or memslice_check):
......@@ -1732,12 +1745,15 @@ class NameNode(AtomicExprNode):
print("NameNode.generate_assignment_code:")
print("...generating post-assignment code for %s" % rhs)
rhs.generate_post_assignment_code(code)
elif rhs.result_in_temp():
rhs.generate_post_assignment_code(code)
rhs.free_temps(code)
def generate_acquire_memoryviewslice(self, rhs, code):
"""
If the value was coerced to a memoryviewslice, its a new reference.
Otherwise we're simply using a borrowed reference from another slice
Slices, coercions from objects, return values etc are new references.
We have a borrowed reference in case of dst = src
"""
import MemoryView
......@@ -1747,7 +1763,8 @@ class NameNode(AtomicExprNode):
lhs_pos=self.pos,
rhs=rhs,
code=code,
incref_rhs=not isinstance(rhs, CoerceToMemViewSliceNode))
incref_rhs=rhs.is_name,
have_gil=not self.nogil)
def generate_acquire_buffer(self, rhs, code):
# rhstmp is only used in case the rhs is a complicated expression leading to
......@@ -2469,9 +2486,7 @@ class IndexNode(ExprNode):
elif index.type.is_int:
self.memslice_index = True
index = index.coerce_to(index_type, env)\
#.coerce_to_temp(
# index_type)
index = index.coerce_to(index_type, env)
indices[i] = index
new_indices.append(index)
......@@ -2488,6 +2503,8 @@ class IndexNode(ExprNode):
self.memslice_index = self.memslice_index and not self.memslice_slice
self.original_indices = indices
# All indices with all start/stop/step for slices.
# We need to keep this around
self.indices = new_indices
self.env = env
......@@ -2540,7 +2557,9 @@ class IndexNode(ExprNode):
error(self.pos, "memoryviews currently support setting only.")
elif self.memslice_slice:
self.index = None
self.is_temp = True
self.use_managed_ref = True
self.type = PyrexTypes.MemoryViewSliceType(
self.base.type.dtype, axes)
......@@ -2618,8 +2637,8 @@ class IndexNode(ExprNode):
gil_message = "Indexing Python object"
def nogil_check(self, env):
if self.is_buffer_access or self.memslice_index:
if env.directives['boundscheck']:
if self.is_buffer_access or self.memslice_index or self.memslice_slice:
if not self.memslice_slice and env.directives['boundscheck']:
error(self.pos, "Cannot check buffer index bounds without gil; use boundscheck(False) directive")
return
elif self.type.is_pyobject:
......@@ -2887,12 +2906,13 @@ class IndexNode(ExprNode):
def put_memoryviewslice_slice_code(self, code):
buffer_entry = self.buffer_entry()
have_gil = not self.in_nogil_context
buffer_entry.generate_buffer_slice_code(code,
self.original_indices,
self.base.type,
self.type,
self.result(),
have_gil = not self.env.nogil)
have_gil=have_gil)
def put_nonecheck(self, code):
code.globalstate.use_utility_code(raise_noneindex_error_utility_code)
......@@ -6206,8 +6226,6 @@ class CythonArrayNode(ExprNode):
import MemoryView
self.type = error_type
self.env = env
self.shapes = []
for axis_no, axis in enumerate(self.base_type_node.axes):
......@@ -8004,8 +8022,9 @@ class CoerceToMemViewSliceNode(CoercionNode):
assert not arg.type.is_memoryviewslice
CoercionNode.__init__(self, arg)
self.type = dst_type
self.env = env
self.is_temp = 1
self.env = env
self.use_managed_ref = True
self.arg = arg
def generate_result_code(self, code):
......@@ -8017,10 +8036,6 @@ class CoerceToMemViewSliceNode(CoercionNode):
error_cond = self.type.error_condition(self.result())
code.putln(code.error_goto_if(error_cond, self.pos))
def generate_disposal_code(self, code):
code.put_xdecref_memoryviewslice(self.result(),
have_gil=not self.env.nogil)
class CastNode(CoercionNode):
# Wrap a node in a C type cast.
......
Cython/Compiler/MemoryView.py
View file @ 2e17dee8
......@@ -79,7 +79,7 @@ def mangle_dtype_name(dtype):
# return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code,
incref_rhs=True, have_gil=False):
incref_rhs=False, have_gil=False):
assert rhs.type.is_memoryviewslice
pretty_rhs = isinstance(rhs, NameNode) or rhs.result_in_temp()
......@@ -97,18 +97,20 @@ def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code,
code.funcstate.release_temp(rhstmp)
def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, code,
incref_rhs=True):
incref_rhs=False):
code.put_xdecref_memoryviewslice(lhs_cname)
if incref_rhs:
code.put_incref_memoryviewslice(rhs_cname)
code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname))
code.putln("%s.data = %s.data;" % (lhs_cname, rhs_cname))
for i in range(memviewslicetype.ndim):
tup = (lhs_cname, i, rhs_cname, i)
code.putln("%s.shape[%d] = %s.shape[%d];" % tup)
code.putln("%s.strides[%d] = %s.strides[%d];" % tup)
code.putln("%s.suboffsets[%d] = %s.suboffsets[%d];" % tup)
code.putln("%s = %s;" % (lhs_cname, rhs_cname))
#code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname))
#code.putln("%s.data = %s.data;" % (lhs_cname, rhs_cname))
#for i in range(memviewslicetype.ndim):
# tup = (lhs_cname, i, rhs_cname, i)
# code.putln("%s.shape[%d] = %s.shape[%d];" % tup)
# code.putln("%s.strides[%d] = %s.strides[%d];" % tup)
# code.putln("%s.suboffsets[%d] = %s.suboffsets[%d];" % tup)
def get_buf_flags(specs):
is_c_contig, is_f_contig = is_cf_contig(specs)
......@@ -264,9 +266,9 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
code.putln( "if (unlikely(__pyx_t_result)) {")
code.put_ensure_gil()
code.putln( "PyErr_Format(PyExc_IndexError, "
"__pyx_t_result, %d)" % dim)
"__pyx_t_result, %d);" % dim)
code.put_release_ensured_gil()
code.putln(code.goto_error(pos))
code.putln(code.error_goto(pos))
code.putln( "}")
code.putln("}")
......@@ -274,6 +276,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
code.putln("%s = -1;" % suboffset_dim)
code.putln("%(dst)s.data = %(cname)s.data;" % locals())
code.putln("%(dst)s.memview = %(cname)s.memview;" % locals())
code.put_incref_memoryviewslice(dst)
for dim, index in enumerate(indices):
if not isinstance(index, ExprNodes.SliceNode):
......@@ -295,6 +298,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
code.funcstate.release_temp(suboffset_dim)
def empty_slice(pos):
none = ExprNodes.NoneNode(pos)
return ExprNodes.SliceNode(pos, start=none,
......
Cython/Compiler/Nodes.py
View file @ 2e17dee8
......@@ -1495,7 +1495,7 @@ class FuncDefNode(StatNode, BlockNode):
code.put_goto(code.return_label)
code.put_label(code.error_label)
for cname, type in code.funcstate.all_managed_temps():
code.put_xdecref(cname, type)
code.put_xdecref(cname, type, have_gil=not lenv.nogil)
# Clean up buffers -- this calls a Python function
# so need to save and restore error state
......@@ -4347,8 +4347,8 @@ class ReturnStatNode(StatNode):
if self.return_type.is_pyobject:
code.put_xdecref(Naming.retval_cname,
self.return_type)
elif self.return_type.is_memoryviewslice:
code.put_xdecref_memoryviewslice(Naming.retval_cname)
#elif self.return_type.is_memoryviewslice:
# code.put_xdecref_memoryviewslice(Naming.retval_cname)
if self.value:
self.value.generate_evaluation_code(code)
......@@ -4359,7 +4359,9 @@ class ReturnStatNode(StatNode):
lhs_type=self.return_type,
lhs_pos=self.value.pos,
rhs=self.value,
code=code)
code=code,
incref_rhs=True,
have_gil=self.in_nogil_context)
else:
self.value.make_owned_reference(code)
code.putln(
......
Cython/Compiler/ParseTreeTransforms.py
View file @ 2e17dee8
......@@ -2223,6 +2223,7 @@ class GilCheck(VisitorTransform):
if self.env_stack and self.nogil and node.nogil_check:
node.nogil_check(self.env_stack[-1])
self.visitchildren(node)
node.in_nogil_context = self.nogil
return node
......
Cython/Utility/MemoryView.pyx
View file @ 2e17dee8
......@@ -460,16 +460,6 @@ cdef extern from "pystate.h":
void PyErr_SetString(PyObject *type, char *msg) nogil
PyObject *PyErr_Format(PyObject *exc, char *msg, ...) nogil
cdef:
char *ERR_OOB = "Index out of bounds (axis %d)"
char *ERR_STEP = "Step must not be zero (axis %d)"
char *ERR_INDIRECT_GIL = ("Cannot make indirect dimension %d disappear "
"through indexing, consider slicing with %d:%d")
char *ERR_INDIRECT_NOGIL = ("Cannot make indirect dimension %d disappear "
"through indexing")
PyObject *exc = <PyObject *> IndexError
@cname('__pyx_memoryview_slice_memviewslice')
cdef char *slice_memviewslice({{memviewslice_name}} *src,
{{memviewslice_name}} *dst,
......@@ -500,6 +490,15 @@ cdef char *slice_memviewslice({{memviewslice_name}} *src,
Py_ssize_t new_shape
bint negative_step
# Somehow these pointers are NULL when set as globals... this needs investigation
char *ERR_OOB = "Index out of bounds (axis %d)"
char *ERR_STEP = "Step must not be zero (axis %d)"
char *ERR_INDIRECT_GIL = ("Cannot make indirect dimension %d disappear "
"through indexing, consider slicing with %d:%d")
char *ERR_INDIRECT_NOGIL = ("Cannot make indirect dimension %d disappear "
"through indexing")
PyObject *exc = <PyObject *> IndexError
if have_gil:
# Assert the GIL
PyThreadState_Get()
......
tests/run/memslice.pyx
View file @ 2e17dee8
......@@ -1330,3 +1330,73 @@ def test_oob():
"""
cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9))
print a[:, 20]
cdef int nogil_oob(int[:, :] a) nogil except 0:
a[100, 9:]
return 1
@testcase
def test_nogil_oob1():
"""
A is acquired at the beginning of the function and released at the end.
B is acquired as a temporary and as such is immediately released in the
except clause.
>>> test_nogil_oob1()
acquired A
acquired B
released B
Index out of bounds (axis 0)
Index out of bounds (axis 0)
released A
"""
cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9))
try:
nogil_oob(IntMockBuffer("B", range(4 * 9), shape=(4, 9)))
except IndexError, e:
print e.args[0]
try:
# Enable when the nogil exception propagation fix is merged
#with nogil:
nogil_oob(a)
except IndexError, e:
print e.args[0]
@testcase
def test_nogil_oob2():
"""
>>> test_nogil_oob2()
Traceback (most recent call last):
...
IndexError: Index out of bounds (axis 0)
"""
cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9))
with nogil:
a[100, 9:]
@cython.boundscheck(False)
cdef int cdef_nogil(int[:, :] a) nogil except 0:
cdef int i, j
cdef int[:, :] b = a[::-1, 3:10:2]
for i in range(b.shape[0]):
for j in range(b.shape[1]):
b[i, j] = -b[i, j]
return 1
@testcase
def test_nogil():
"""
>>> test_nogil()
acquired A
released A
acquired A
-25
released A
"""
_a = IntMockBuffer("A", range(4 * 9), shape=(4, 9))
cdef_nogil(_a)
cdef int[:, :] a = _a
print a[2, 7]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7