Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
C
cython
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Kirill Smelkov
cython
Commits
2e17dee8
Commit
2e17dee8
authored
Aug 20, 2011
by
Mark Florisson
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Memslice nogil tests, better acquisition counting
parent
b7e14b9a
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
154 additions
and
52 deletions
+154
-52
Cython/Compiler/Code.py
Cython/Compiler/Code.py
+20
-9
Cython/Compiler/ExprNodes.py
Cython/Compiler/ExprNodes.py
+33
-18
Cython/Compiler/MemoryView.py
Cython/Compiler/MemoryView.py
+15
-11
Cython/Compiler/Nodes.py
Cython/Compiler/Nodes.py
+6
-4
Cython/Compiler/ParseTreeTransforms.py
Cython/Compiler/ParseTreeTransforms.py
+1
-0
Cython/Utility/MemoryView.pyx
Cython/Utility/MemoryView.pyx
+9
-10
tests/run/memslice.pyx
tests/run/memslice.pyx
+70
-0
No files found.
Cython/Compiler/Code.py
View file @
2e17dee8
...
...
@@ -462,10 +462,11 @@ class FunctionState(object):
A C string referring to the variable is returned.
"""
if
not
type
.
is_pyobject
:
if
not
type
.
is_pyobject
and
not
type
.
is_memoryviewslice
:
# Make manage_ref canonical, so that manage_ref will always mean
# a decref is needed.
manage_ref
=
False
freelist
=
self
.
temps_free
.
get
((
type
,
manage_ref
))
if
freelist
is
not
None
and
len
(
freelist
)
>
0
:
result
=
freelist
.
pop
()
...
...
@@ -508,7 +509,7 @@ class FunctionState(object):
for
name
,
type
,
manage_ref
in
self
.
temps_allocated
:
freelist
=
self
.
temps_free
.
get
((
type
,
manage_ref
))
if
freelist
is
None
or
name
not
in
freelist
:
used
.
append
((
name
,
type
,
manage_ref
))
used
.
append
((
name
,
type
,
manage_ref
and
type
.
is_pyobject
))
return
used
def
temps_holding_reference
(
self
):
...
...
@@ -518,14 +519,14 @@ class FunctionState(object):
"""
return
[(
name
,
type
)
for
name
,
type
,
manage_ref
in
self
.
temps_in_use
()
if
manage_ref
]
if
manage_ref
and
type
.
is_pyobject
]
def
all_managed_temps
(
self
):
"""Return a list of (cname, type) tuples of refcount-managed Python objects.
"""
return
[(
cname
,
type
)
for
cname
,
type
,
manage_ref
in
self
.
temps_allocated
if
manage_ref
]
for
cname
,
type
,
manage_ref
in
self
.
temps_allocated
if
manage_ref
]
def
all_free_managed_temps
(
self
):
"""Return a list of (cname, type) tuples of refcount-managed Python
...
...
@@ -534,9 +535,9 @@ class FunctionState(object):
error case.
"""
return
[(
cname
,
type
)
for
(
type
,
manage_ref
),
freelist
in
self
.
temps_free
.
items
()
if
manage_ref
for
cname
in
freelist
]
for
(
type
,
manage_ref
),
freelist
in
self
.
temps_free
.
items
()
if
manage_ref
for
cname
in
freelist
]
def
start_collecting_temps
(
self
):
"""
...
...
@@ -1464,6 +1465,8 @@ class CCodeWriter(object):
decl
=
type
.
declaration_code
(
name
)
if
type
.
is_pyobject
:
self
.
putln
(
"%s = NULL;"
%
decl
)
elif
type
.
is_memoryviewslice
:
self
.
putln
(
"%s = { 0 };"
%
decl
)
else
:
self
.
putln
(
"%s;"
%
decl
)
...
...
@@ -1544,13 +1547,21 @@ class CCodeWriter(object):
self
.
putln
(
"Py_DECREF(%s); %s = 0;"
%
(
typecast
(
py_object_type
,
type
,
cname
),
cname
))
def
put_xdecref
(
self
,
cname
,
type
,
nanny
=
True
):
def
put_xdecref
(
self
,
cname
,
type
,
nanny
=
True
,
have_gil
=
True
):
if
type
.
is_memoryviewslice
:
self
.
put_xdecref_memoryviewslice
(
cname
,
have_gil
=
have_gil
)
return
if
nanny
:
self
.
putln
(
"__Pyx_XDECREF(%s);"
%
self
.
as_pyobject
(
cname
,
type
))
else
:
self
.
putln
(
"Py_XDECREF(%s);"
%
self
.
as_pyobject
(
cname
,
type
))
def
put_xdecref_clear
(
self
,
cname
,
type
,
nanny
=
True
):
if
type
.
is_memoryviewslice
:
self
.
put_xdecref_memoryviewslice
(
cname
)
return
if
nanny
:
self
.
putln
(
"__Pyx_XDECREF(%s); %s = 0;"
%
(
self
.
as_pyobject
(
cname
,
type
),
cname
))
...
...
Cython/Compiler/ExprNodes.py
View file @
2e17dee8
...
...
@@ -504,8 +504,12 @@ class ExprNode(Node):
def
generate_disposal_code
(
self
,
code
):
if
self
.
is_temp
:
if
self
.
type
.
is_pyobject
and
self
.
result
():
code
.
put_decref_clear
(
self
.
result
(),
self
.
ctype
())
if
self
.
result
():
if
self
.
type
.
is_pyobject
:
code
.
put_decref_clear
(
self
.
result
(),
self
.
ctype
())
elif
self
.
type
.
is_memoryviewslice
:
code
.
put_xdecref_memoryviewslice
(
self
.
result
(),
have_gil
=
not
self
.
in_nogil_context
)
else
:
# Already done if self.is_temp
self
.
generate_subexpr_disposal_code
(
code
)
...
...
@@ -520,6 +524,9 @@ class ExprNode(Node):
if
self
.
is_temp
:
if
self
.
type
.
is_pyobject
:
code
.
putln
(
"%s = 0;"
%
self
.
result
())
elif
self
.
type
.
is_memoryviewslice
:
code
.
putln
(
"%s.memview = NULL;"
%
self
.
result
())
code
.
putln
(
"%s.data = NULL;"
%
self
.
result
())
else
:
self
.
generate_subexpr_disposal_code
(
code
)
...
...
@@ -1477,6 +1484,7 @@ class NameNode(AtomicExprNode):
elif
entry
.
type
.
is_memoryviewslice
:
self
.
is_temp
=
False
self
.
is_used_as_rvalue
=
True
self
.
use_managed_ref
=
True
def
nogil_check
(
self
,
env
):
self
.
nogil
=
True
...
...
@@ -1635,6 +1643,11 @@ class NameNode(AtomicExprNode):
raise_unbound
=
(
(
self
.
cf_maybe_null
or
self
.
cf_is_null
)
and
not
self
.
allow_null
)
null_code
=
entry
.
type
.
check_for_null_code
(
entry
.
cname
)
# if entry.type.is_memoryviewslice:
# have_gil = not self.in_nogil_context
# code.put_incref_memoryviewslice(entry.cname, have_gil)
memslice_check
=
entry
.
type
.
is_memoryviewslice
and
self
.
initialized_check
if
null_code
and
raise_unbound
and
(
entry
.
type
.
is_pyobject
or
memslice_check
):
...
...
@@ -1732,12 +1745,15 @@ class NameNode(AtomicExprNode):
print
(
"NameNode.generate_assignment_code:"
)
print
(
"...generating post-assignment code for %s"
%
rhs
)
rhs
.
generate_post_assignment_code
(
code
)
elif
rhs
.
result_in_temp
():
rhs
.
generate_post_assignment_code
(
code
)
rhs
.
free_temps
(
code
)
def
generate_acquire_memoryviewslice
(
self
,
rhs
,
code
):
"""
If the value was coerced to a memoryviewslice, its a new reference
.
Otherwise we're simply using a borrowed reference from another slice
Slices, coercions from objects, return values etc are new references
.
We have a borrowed reference in case of dst = src
"""
import
MemoryView
...
...
@@ -1747,7 +1763,8 @@ class NameNode(AtomicExprNode):
lhs_pos
=
self
.
pos
,
rhs
=
rhs
,
code
=
code
,
incref_rhs
=
not
isinstance
(
rhs
,
CoerceToMemViewSliceNode
))
incref_rhs
=
rhs
.
is_name
,
have_gil
=
not
self
.
nogil
)
def
generate_acquire_buffer
(
self
,
rhs
,
code
):
# rhstmp is only used in case the rhs is a complicated expression leading to
...
...
@@ -2469,9 +2486,7 @@ class IndexNode(ExprNode):
elif
index
.
type
.
is_int
:
self
.
memslice_index
=
True
index
=
index
.
coerce_to
(
index_type
,
env
)
\
#.coerce_to_temp(
# index_type)
index
=
index
.
coerce_to
(
index_type
,
env
)
indices
[
i
]
=
index
new_indices
.
append
(
index
)
...
...
@@ -2488,6 +2503,8 @@ class IndexNode(ExprNode):
self
.
memslice_index
=
self
.
memslice_index
and
not
self
.
memslice_slice
self
.
original_indices
=
indices
# All indices with all start/stop/step for slices.
# We need to keep this around
self
.
indices
=
new_indices
self
.
env
=
env
...
...
@@ -2540,7 +2557,9 @@ class IndexNode(ExprNode):
error
(
self
.
pos
,
"memoryviews currently support setting only."
)
elif
self
.
memslice_slice
:
self
.
index
=
None
self
.
is_temp
=
True
self
.
use_managed_ref
=
True
self
.
type
=
PyrexTypes
.
MemoryViewSliceType
(
self
.
base
.
type
.
dtype
,
axes
)
...
...
@@ -2618,8 +2637,8 @@ class IndexNode(ExprNode):
gil_message
=
"Indexing Python object"
def
nogil_check
(
self
,
env
):
if
self
.
is_buffer_access
or
self
.
memslice_index
:
if
env
.
directives
[
'boundscheck'
]:
if
self
.
is_buffer_access
or
self
.
memslice_index
or
self
.
memslice_slice
:
if
not
self
.
memslice_slice
and
env
.
directives
[
'boundscheck'
]:
error
(
self
.
pos
,
"Cannot check buffer index bounds without gil; use boundscheck(False) directive"
)
return
elif
self
.
type
.
is_pyobject
:
...
...
@@ -2887,12 +2906,13 @@ class IndexNode(ExprNode):
def
put_memoryviewslice_slice_code
(
self
,
code
):
buffer_entry
=
self
.
buffer_entry
()
have_gil
=
not
self
.
in_nogil_context
buffer_entry
.
generate_buffer_slice_code
(
code
,
self
.
original_indices
,
self
.
base
.
type
,
self
.
type
,
self
.
result
(),
have_gil
=
not
self
.
env
.
no
gil
)
have_gil
=
have_
gil
)
def
put_nonecheck
(
self
,
code
):
code
.
globalstate
.
use_utility_code
(
raise_noneindex_error_utility_code
)
...
...
@@ -6206,8 +6226,6 @@ class CythonArrayNode(ExprNode):
import
MemoryView
self
.
type
=
error_type
self
.
env
=
env
self
.
shapes
=
[]
for
axis_no
,
axis
in
enumerate
(
self
.
base_type_node
.
axes
):
...
...
@@ -8004,8 +8022,9 @@ class CoerceToMemViewSliceNode(CoercionNode):
assert
not
arg
.
type
.
is_memoryviewslice
CoercionNode
.
__init__
(
self
,
arg
)
self
.
type
=
dst_type
self
.
env
=
env
self
.
is_temp
=
1
self
.
env
=
env
self
.
use_managed_ref
=
True
self
.
arg
=
arg
def
generate_result_code
(
self
,
code
):
...
...
@@ -8017,10 +8036,6 @@ class CoerceToMemViewSliceNode(CoercionNode):
error_cond
=
self
.
type
.
error_condition
(
self
.
result
())
code
.
putln
(
code
.
error_goto_if
(
error_cond
,
self
.
pos
))
def
generate_disposal_code
(
self
,
code
):
code
.
put_xdecref_memoryviewslice
(
self
.
result
(),
have_gil
=
not
self
.
env
.
nogil
)
class
CastNode
(
CoercionNode
):
# Wrap a node in a C type cast.
...
...
Cython/Compiler/MemoryView.py
View file @
2e17dee8
...
...
@@ -79,7 +79,7 @@ def mangle_dtype_name(dtype):
# return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
def
put_acquire_memoryviewslice
(
lhs_cname
,
lhs_type
,
lhs_pos
,
rhs
,
code
,
incref_rhs
=
Tru
e
,
have_gil
=
False
):
incref_rhs
=
Fals
e
,
have_gil
=
False
):
assert
rhs
.
type
.
is_memoryviewslice
pretty_rhs
=
isinstance
(
rhs
,
NameNode
)
or
rhs
.
result_in_temp
()
...
...
@@ -97,18 +97,20 @@ def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code,
code
.
funcstate
.
release_temp
(
rhstmp
)
def
put_assign_to_memviewslice
(
lhs_cname
,
rhs_cname
,
memviewslicetype
,
code
,
incref_rhs
=
Tru
e
):
incref_rhs
=
Fals
e
):
code
.
put_xdecref_memoryviewslice
(
lhs_cname
)
if
incref_rhs
:
code
.
put_incref_memoryviewslice
(
rhs_cname
)
code
.
putln
(
"%s.memview = %s.memview;"
%
(
lhs_cname
,
rhs_cname
))
code
.
putln
(
"%s.data = %s.data;"
%
(
lhs_cname
,
rhs_cname
))
for
i
in
range
(
memviewslicetype
.
ndim
):
tup
=
(
lhs_cname
,
i
,
rhs_cname
,
i
)
code
.
putln
(
"%s.shape[%d] = %s.shape[%d];"
%
tup
)
code
.
putln
(
"%s.strides[%d] = %s.strides[%d];"
%
tup
)
code
.
putln
(
"%s.suboffsets[%d] = %s.suboffsets[%d];"
%
tup
)
code
.
putln
(
"%s = %s;"
%
(
lhs_cname
,
rhs_cname
))
#code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname))
#code.putln("%s.data = %s.data;" % (lhs_cname, rhs_cname))
#for i in range(memviewslicetype.ndim):
# tup = (lhs_cname, i, rhs_cname, i)
# code.putln("%s.shape[%d] = %s.shape[%d];" % tup)
# code.putln("%s.strides[%d] = %s.strides[%d];" % tup)
# code.putln("%s.suboffsets[%d] = %s.suboffsets[%d];" % tup)
def
get_buf_flags
(
specs
):
is_c_contig
,
is_f_contig
=
is_cf_contig
(
specs
)
...
...
@@ -264,9 +266,9 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
code
.
putln
(
"if (unlikely(__pyx_t_result)) {"
)
code
.
put_ensure_gil
()
code
.
putln
(
"PyErr_Format(PyExc_IndexError, "
"__pyx_t_result, %d)"
%
dim
)
"__pyx_t_result, %d)
;
"
%
dim
)
code
.
put_release_ensured_gil
()
code
.
putln
(
code
.
goto_error
(
pos
))
code
.
putln
(
code
.
error_goto
(
pos
))
code
.
putln
(
"}"
)
code
.
putln
(
"}"
)
...
...
@@ -274,6 +276,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
code
.
putln
(
"%s = -1;"
%
suboffset_dim
)
code
.
putln
(
"%(dst)s.data = %(cname)s.data;"
%
locals
())
code
.
putln
(
"%(dst)s.memview = %(cname)s.memview;"
%
locals
())
code
.
put_incref_memoryviewslice
(
dst
)
for
dim
,
index
in
enumerate
(
indices
):
if
not
isinstance
(
index
,
ExprNodes
.
SliceNode
):
...
...
@@ -295,6 +298,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
code
.
funcstate
.
release_temp
(
suboffset_dim
)
def
empty_slice
(
pos
):
none
=
ExprNodes
.
NoneNode
(
pos
)
return
ExprNodes
.
SliceNode
(
pos
,
start
=
none
,
...
...
Cython/Compiler/Nodes.py
View file @
2e17dee8
...
...
@@ -1495,7 +1495,7 @@ class FuncDefNode(StatNode, BlockNode):
code
.
put_goto
(
code
.
return_label
)
code
.
put_label
(
code
.
error_label
)
for
cname
,
type
in
code
.
funcstate
.
all_managed_temps
():
code
.
put_xdecref
(
cname
,
type
)
code
.
put_xdecref
(
cname
,
type
,
have_gil
=
not
lenv
.
nogil
)
# Clean up buffers -- this calls a Python function
# so need to save and restore error state
...
...
@@ -4347,8 +4347,8 @@ class ReturnStatNode(StatNode):
if
self
.
return_type
.
is_pyobject
:
code
.
put_xdecref
(
Naming
.
retval_cname
,
self
.
return_type
)
elif
self
.
return_type
.
is_memoryviewslice
:
code
.
put_xdecref_memoryviewslice
(
Naming
.
retval_cname
)
#
elif self.return_type.is_memoryviewslice:
#
code.put_xdecref_memoryviewslice(Naming.retval_cname)
if
self
.
value
:
self
.
value
.
generate_evaluation_code
(
code
)
...
...
@@ -4359,7 +4359,9 @@ class ReturnStatNode(StatNode):
lhs_type
=
self
.
return_type
,
lhs_pos
=
self
.
value
.
pos
,
rhs
=
self
.
value
,
code
=
code
)
code
=
code
,
incref_rhs
=
True
,
have_gil
=
self
.
in_nogil_context
)
else
:
self
.
value
.
make_owned_reference
(
code
)
code
.
putln
(
...
...
Cython/Compiler/ParseTreeTransforms.py
View file @
2e17dee8
...
...
@@ -2223,6 +2223,7 @@ class GilCheck(VisitorTransform):
if
self
.
env_stack
and
self
.
nogil
and
node
.
nogil_check
:
node
.
nogil_check
(
self
.
env_stack
[
-
1
])
self
.
visitchildren
(
node
)
node
.
in_nogil_context
=
self
.
nogil
return
node
...
...
Cython/Utility/MemoryView.pyx
View file @
2e17dee8
...
...
@@ -460,16 +460,6 @@ cdef extern from "pystate.h":
void
PyErr_SetString
(
PyObject
*
type
,
char
*
msg
)
nogil
PyObject
*
PyErr_Format
(
PyObject
*
exc
,
char
*
msg
,
...)
nogil
cdef
:
char
*
ERR_OOB
=
"Index out of bounds (axis %d)"
char
*
ERR_STEP
=
"Step must not be zero (axis %d)"
char
*
ERR_INDIRECT_GIL
=
(
"Cannot make indirect dimension %d disappear "
"through indexing, consider slicing with %d:%d"
)
char
*
ERR_INDIRECT_NOGIL
=
(
"Cannot make indirect dimension %d disappear "
"through indexing"
)
PyObject
*
exc
=
<
PyObject
*>
IndexError
@
cname
(
'__pyx_memoryview_slice_memviewslice'
)
cdef
char
*
slice_memviewslice
({{
memviewslice_name
}}
*
src
,
{{
memviewslice_name
}}
*
dst
,
...
...
@@ -500,6 +490,15 @@ cdef char *slice_memviewslice({{memviewslice_name}} *src,
Py_ssize_t
new_shape
bint
negative_step
# Somehow these pointers are NULL when set as globals... this needs investigation
char
*
ERR_OOB
=
"Index out of bounds (axis %d)"
char
*
ERR_STEP
=
"Step must not be zero (axis %d)"
char
*
ERR_INDIRECT_GIL
=
(
"Cannot make indirect dimension %d disappear "
"through indexing, consider slicing with %d:%d"
)
char
*
ERR_INDIRECT_NOGIL
=
(
"Cannot make indirect dimension %d disappear "
"through indexing"
)
PyObject
*
exc
=
<
PyObject
*>
IndexError
if
have_gil
:
# Assert the GIL
PyThreadState_Get
()
...
...
tests/run/memslice.pyx
View file @
2e17dee8
...
...
@@ -1330,3 +1330,73 @@ def test_oob():
"""
cdef
int
[:,
:]
a
=
IntMockBuffer
(
"A"
,
range
(
4
*
9
),
shape
=
(
4
,
9
))
print
a
[:,
20
]
cdef
int
nogil_oob
(
int
[:,
:]
a
)
nogil
except
0
:
a
[
100
,
9
:]
return
1
@
testcase
def
test_nogil_oob1
():
"""
A is acquired at the beginning of the function and released at the end.
B is acquired as a temporary and as such is immediately released in the
except clause.
>>> test_nogil_oob1()
acquired A
acquired B
released B
Index out of bounds (axis 0)
Index out of bounds (axis 0)
released A
"""
cdef
int
[:,
:]
a
=
IntMockBuffer
(
"A"
,
range
(
4
*
9
),
shape
=
(
4
,
9
))
try
:
nogil_oob
(
IntMockBuffer
(
"B"
,
range
(
4
*
9
),
shape
=
(
4
,
9
)))
except
IndexError
,
e
:
print
e
.
args
[
0
]
try
:
# Enable when the nogil exception propagation fix is merged
#with nogil:
nogil_oob
(
a
)
except
IndexError
,
e
:
print
e
.
args
[
0
]
@
testcase
def
test_nogil_oob2
():
"""
>>> test_nogil_oob2()
Traceback (most recent call last):
...
IndexError: Index out of bounds (axis 0)
"""
cdef
int
[:,
:]
a
=
IntMockBuffer
(
"A"
,
range
(
4
*
9
),
shape
=
(
4
,
9
))
with
nogil
:
a
[
100
,
9
:]
@
cython
.
boundscheck
(
False
)
cdef
int
cdef
_nogil
(
int
[:,
:]
a
)
nogil
except
0
:
cdef
int
i
,
j
cdef
int
[:,
:]
b
=
a
[::
-
1
,
3
:
10
:
2
]
for
i
in
range
(
b
.
shape
[
0
]):
for
j
in
range
(
b
.
shape
[
1
]):
b
[
i
,
j
]
=
-
b
[
i
,
j
]
return
1
@
testcase
def
test_nogil
():
"""
>>> test_nogil()
acquired A
released A
acquired A
-25
released A
"""
_a
=
IntMockBuffer
(
"A"
,
range
(
4
*
9
),
shape
=
(
4
,
9
))
cdef
_nogil
(
_a
)
cdef
int
[:,
:]
a
=
_a
print
a
[
2
,
7
]
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment