Skip to content

C
cython
Commit b7e14b9a authored by Mark Florisson's avatar Mark Florisson
Browse files
Options

cimport cython in CyUtility, more tests, nogil slicing

parent 8105941c
Expand all Show whitespace changes
Inline Side-by-side
Showing with 561 additions and 340 deletions
Cython/Compiler/Code.pxd
View file @ b7e14b9a
......@@ -13,6 +13,8 @@ cdef class UtilityCode(UtilityCodeBase):
cdef public dict _cache
cdef public list specialize_list
cdef public object proto_block
cdef public object name
cdef public object file
cpdef put_code(self, output)
......
Cython/Compiler/CythonScope.py
View file @ b7e14b9a
......@@ -75,10 +75,13 @@ class CythonScope(ModuleScope):
Creates some entries for testing purposes and entries for
cython.array() and for cython.view.*.
"""
cython_testscope_utility_code.declare_in_scope(self)
cython_test_extclass_utility_code.declare_in_scope(self)
cython_testscope_utility_code.declare_in_scope(
self, cython_scope=self)
cython_test_extclass_utility_code.declare_in_scope(
self, cython_scope=self)
MemoryView.cython_array_utility_code.declare_in_scope(self)
MemoryView.cython_array_utility_code.declare_in_scope(
self, cython_scope=self)
#
# The view sub-scope
......@@ -88,9 +91,11 @@ class CythonScope(ModuleScope):
viewscope.is_cython_builtin = True
viewscope.pxd_file_loaded = True
cythonview_testscope_utility_code.declare_in_scope(viewscope)
cythonview_testscope_utility_code.declare_in_scope(
viewscope, cython_scope=self)
view_utility_scope = MemoryView.view_utility_code.declare_in_scope(viewscope)
view_utility_scope = MemoryView.view_utility_code.declare_in_scope(
viewscope, cython_scope=self)
# MemoryView.memview_fromslice_utility_code.from_scope = view_utility_scope
# MemoryView.memview_fromslice_utility_code.declare_in_scope(viewscope)
......
Cython/Compiler/ExprNodes.py
View file @ b7e14b9a
......@@ -2432,7 +2432,8 @@ class IndexNode(ExprNode):
import MemoryView
skip_child_analysis = True
indices = MemoryView.unellipsify(indices, self.base.type.ndim)
have_slices, indices = MemoryView.unellipsify(indices,
self.base.type.ndim)
self.memslice_index = len(indices) == self.base.type.ndim
axes = []
......@@ -2462,24 +2463,19 @@ class IndexNode(ExprNode):
value = getattr(index, attr)
if not value.is_none:
value = value.coerce_to(index_type, env)
value = value.coerce_to_temp(env)
#value = value.coerce_to_temp(env)
setattr(index, attr, value)
new_indices.append(value)
elif index.type.is_int:
self.memslice_index = True
index = index.coerce_to(index_type, env).coerce_to_temp(
index_type)
index = index.coerce_to(index_type, env)\
#.coerce_to_temp(
# index_type)
indices[i] = index
new_indices.append(index)
if access in ('ptr', 'generic') and i != 0:
# If this dimension is to disappear, then how do we
# indicate that we need to dereference in this dimension
# if the previous dimension is already indirect, or if
# the previous dimension was direct but also indexed?
# Basically only a[i, j, k, :] can work, as you can
# set the base pointer to start in the fourth dimension
if access in ('ptr', 'generic') and i != 0 and have_slices:
self.type = error_type
return error(index.pos,
"Indexing of non-leading indirect or generic "
......@@ -2494,6 +2490,8 @@ class IndexNode(ExprNode):
self.original_indices = indices
self.indices = new_indices
self.env = env
elif self.base.type.is_buffer:
# Buffer indexing
if len(indices) == self.base.type.ndim:
......@@ -2893,7 +2891,8 @@ class IndexNode(ExprNode):
self.original_indices,
self.base.type,
self.type,
self.result())
self.result(),
have_gil = not self.env.nogil)
def put_nonecheck(self, code):
code.globalstate.use_utility_code(raise_noneindex_error_utility_code)
......@@ -4092,6 +4091,7 @@ class AttributeNode(ExprNode):
def nogil_check(self, env):
if self.is_py_attr:
self.gil_error()
elif self.type.is_memoryviewslice:
import MemoryView
MemoryView.err_if_nogil_initialized_check(self.pos, env, 'attribute')
......
Cython/Compiler/MemoryView.py
View file @ b7e14b9a
This diff is collapsed.
Cython/Compiler/UtilityCode.py
View file @ b7e14b9a
......@@ -9,6 +9,7 @@ class NonManglingModuleScope(Symtab.ModuleScope):
def __init__(self, prefix, *args, **kw):
self.prefix = prefix
self.cython_scope = None
Symtab.ModuleScope.__init__(self, *args, **kw)
def add_imported_entry(self, name, entry, pos):
......@@ -31,9 +32,12 @@ class CythonUtilityCodeContext(StringParseContext):
def find_module(self, module_name, relative_to = None, pos = None,
need_pxd = 1):
if module_name != self.module_name:
raise AssertionError("Not yet supporting any cimports/includes "
"from string code snippets")
if module_name not in self.modules:
raise AssertionError("Only the cython cimport is supported.")
else:
return self.modules[module_name]
if self.scope is None:
self.scope = NonManglingModuleScope(self.prefix,
......@@ -78,7 +82,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
self.requires = requires or []
self.from_scope = from_scope
def get_tree(self, entries_only=False):
def get_tree(self, entries_only=False, cython_scope=None):
from AnalysedTreeTransforms import AutoTestDictTransform
# The AutoTestDictTransform creates the statement "__test__ = {}",
# which when copied into the main ModuleNode overwrites
......@@ -88,6 +92,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
import Pipeline, ParseTreeTransforms
context = CythonUtilityCodeContext(self.name)
context.prefix = self.prefix
context.cython_scope = cython_scope
#context = StringParseContext(self.name)
tree = parse_from_strings(self.name, self.impl, context=context,
allow_struct_enum_decorator=True)
......@@ -125,13 +130,13 @@ class CythonUtilityCode(Code.UtilityCodeBase):
def put_code(self, output):
pass
def declare_in_scope(self, dest_scope, used=False):
def declare_in_scope(self, dest_scope, used=False, cython_scope=None):
"""
Declare all entries from the utility code in dest_scope. Code will only
be included for used entries. If module_name is given, declare the
type entries with that name.
"""
tree = self.get_tree(entries_only=True)
tree = self.get_tree(entries_only=True, cython_scope=cython_scope)
entries = tree.scope.entries
entries.pop('__name__')
......
Cython/Utility/MemoryView.pyx
View file @ b7e14b9a
This diff is collapsed.
Cython/Utility/MemoryView_C.c
View file @ b7e14b9a
......@@ -284,7 +284,7 @@ static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
if (!memview)
return; /* allow uninitialized memoryview assignment */
if (memview->acquisition_count <= 0)
if (memview->acquisition_count < 0)
__pyx_fatalerror("Acquisition count is %d (line %d)",
memview->acquisition_count, lineno);
......
tests/run/memoryview.pyx
View file @ b7e14b9a
......@@ -588,40 +588,122 @@ def assign_temporary_to_object(object[:] mslice):
buf = mslice
buf[1] = {3-2: 2+(2*4)-2}
def test_slicing(arg):
def print_int_offsets(*args):
for item in args:
print item / sizeof(int),
print
def test_generic_slicing(arg):
"""
Test simple slicing
>>> test_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
>>> test_generic_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
acquired A
3 9 2
1232 -44 4
(3, 9, 2)
308 -11 1
-1 -1 -1
released A
Test direct slicing, negative slice oob in dim 2
>>> test_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
>>> test_generic_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
acquired A
0 0 2
48 -12 4
(0, 0, 2)
12 -3 1
-1 -1 -1
released A
Test indirect slicing
>>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
>>> test_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
>>> test_generic_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
acquired A
2 0 2
8 -4 4
0 0 -1
(2, 0, 2)
0 1 -1
released A
>>> stride1 = 21 * 14
>>> stride2 = 21
>>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
>>> test_generic_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
acquired A
(3, 9, 2)
20 1 -1
released A
"""
cdef int[::view.generic, ::view.generic, :] _a = arg
a = _a
b = a[2:8:2, -4:1:-1, 1:3]
print b.shape[0], b.shape[1], b.shape[2]
print b.strides[0], b.strides[1], b.strides[2]
print b.suboffsets[0], b.suboffsets[1], b.suboffsets[2]
print b.shape
if b.suboffsets[0] < 0:
print_int_offsets(*b.strides)
print_int_offsets(*b.suboffsets)
cdef int i, j, k
for i in range(b.shape[0]):
for j in range(b.shape[1]):
for k in range(b.shape[2]):
itemA = a[2 + 2 * i, -4 - j, 1 + k]
itemB = b[i, j, k]
assert itemA == itemB, (i, j, k, itemA, itemB)
def test_indirect_slicing(arg):
"""
Test indirect slicing
>>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
>>> test_indirect_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
acquired A
(5, 3, 2)
0 0 -1
58
released A
>>> stride1 = 21 * 14
>>> stride2 = 21
>>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
>>> test_indirect_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
acquired A
(5, 14, 3)
0 16 -1
2412
released A
"""
cdef int[::view.indirect, ::view.indirect, :] _a = arg
a = _a
b = a[-5:, ..., -5:100:2]
print b.shape
print_int_offsets(*b.suboffsets)
print b[4, 2, 1]
def test_direct_slicing(arg):
"""
Fused types would be convenient to test this stuff!
Test simple slicing
>>> test_direct_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
acquired A
(3, 9, 2)
308 -11 1
-1 -1 -1
released A
Test direct slicing, negative slice oob in dim 2
>>> test_direct_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
acquired A
(0, 0, 2)
12 -3 1
-1 -1 -1
released A
"""
cdef int[:, :, :] _a = arg
a = _a
b = a[2:8:2, -4:1:-1, 1:3]
print b.shape
print_int_offsets(*b.strides)
print_int_offsets(*b.suboffsets)
cdef int i, j, k
for i in range(b.shape[0]):
......@@ -631,13 +713,14 @@ def test_slicing(arg):
itemB = b[i, j, k]
assert itemA == itemB, (i, j, k, itemA, itemB)
def test_slicing_and_indexing(arg):
"""
>>> a = IntStridedMockBuffer("A", range(10 * 3 * 5), shape=(10, 3, 5))
>>> test_slicing_and_indexing(a)
acquired A
5 2
60 8
(5, 2)
15 2
126 113
[111]
released A
......@@ -648,8 +731,8 @@ def test_slicing_and_indexing(arg):
c = b[4:1:-1, ::-1]
d = c[2, 1:2]
print b.shape[0], b.shape[1]
print b.strides[0], b.strides[1]
print b.shape
print_int_offsets(*b.strides)
cdef int i, j
for i in range(b.shape[0]):
......@@ -660,3 +743,13 @@ def test_slicing_and_indexing(arg):
print c[1, 1], c[2, 0]
print [d[i] for i in range(d.shape[0])]
def test_oob():
"""
>>> test_oob()
Traceback (most recent call last):
...
IndexError: Index out of bounds (axis 1)
"""
cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9))
print a[:, 20]
tests/run/memslice.pyx
View file @ b7e14b9a
......@@ -2,6 +2,7 @@
from __future__ import unicode_literals
cimport cython
from cython cimport view
__test__ = {}
......@@ -1158,40 +1159,127 @@ def test_cdef_function2():
cdef_function2(global_A, global_B)
def print_int_offsets(*args):
for item in args:
print item / sizeof(int),
print
@testcase
def test_slicing(arg):
def test_generic_slicing(arg):
"""
Test simple slicing
>>> test_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
>>> test_generic_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
acquired A
3 9 2
1232 -44 4
308 -11 1
-1 -1 -1
released A
Test direct slicing, negative slice oob in dim 2
>>> test_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
>>> test_generic_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
acquired A
0 0 2
48 -12 4
12 -3 1
-1 -1 -1
released A
Test indirect slicing
>>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
>>> test_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
>>> test_generic_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
acquired A
2 0 2
8 -4 4
0 0 -1
0 1 -1
released A
>>> stride1 = 21 * 14
>>> stride2 = 21
>>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
>>> test_generic_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
acquired A
3 9 2
20 1 -1
released A
"""
cdef int[::view.generic, ::view.generic, :] a = arg
cdef int[::view.generic, ::view.generic, :] b = a[2:8:2, -4:1:-1, 1:3]
print b.shape[0], b.shape[1], b.shape[2]
print b.strides[0], b.strides[1], b.strides[2]
print b.suboffsets[0], b.suboffsets[1], b.suboffsets[2]
if b.suboffsets[0] < 0:
print_int_offsets(b.strides[0], b.strides[1], b.strides[2])
print_int_offsets(b.suboffsets[0], b.suboffsets[1], b.suboffsets[2])
cdef int i, j, k
for i in range(b.shape[0]):
for j in range(b.shape[1]):
for k in range(b.shape[2]):
itemA = a[2 + 2 * i, -4 - j, 1 + k]
itemB = b[i, j, k]
assert itemA == itemB, (i, j, k, itemA, itemB)
@testcase
def test_indirect_slicing(arg):
"""
Test indirect slicing
>>> L = [[range(k * 12 + j * 4, k * 12 + j * 4 + 4) for j in xrange(3)] for k in xrange(5)]
>>> test_indirect_slicing(IntMockBuffer("A", L, shape=(5, 3, 4)))
acquired A
5 3 2
0 0 -1
58
56
released A
>>> stride1 = 21 * 14
>>> stride2 = 21
>>> L = [[range(k * stride1 + j * stride2, k * stride1 + j * stride2 + 21) for j in xrange(14)] for k in xrange(9)]
>>> test_indirect_slicing(IntMockBuffer("A", L, shape=(9, 14, 21)))
acquired A
5 14 3
0 16 -1
2412
2410
released A
"""
cdef int[::view.indirect, ::view.indirect, :] a = arg
cdef int[::view.indirect, ::view.indirect, :] b = a[-5:, ..., -5:100:2]
cdef int[::view.indirect, ::view.indirect] c = b[..., 0]
print b.shape[0], b.shape[1], b.shape[2]
print_int_offsets(b.suboffsets[0], b.suboffsets[1], b.suboffsets[2])
print b[4, 2, 1]
print c[4, 2]
@testcase
def test_direct_slicing(arg):
"""
Fused types would be convenient to test this stuff!
Test simple slicing
>>> test_direct_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11)))
acquired A
3 9 2
308 -11 1
-1 -1 -1
released A
Test direct slicing, negative slice oob in dim 2
>>> test_direct_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3)))
acquired A
0 0 2
12 -3 1
-1 -1 -1
released A
"""
cdef int[:, :, ::1] a = arg
cdef int[:, :, :] b = a[2:8:2, -4:1:-1, 1:3]
print b.shape[0], b.shape[1], b.shape[2]
print_int_offsets(b.strides[0], b.strides[1], b.strides[2])
print_int_offsets(b.suboffsets[0], b.suboffsets[1], b.suboffsets[2])
cdef int i, j, k
for i in range(b.shape[0]):
......@@ -1208,7 +1296,7 @@ def test_slicing_and_indexing(arg):
>>> test_slicing_and_indexing(a)
acquired A
5 2
60 8
15 2
126 113
[111]
released A
......@@ -1219,7 +1307,7 @@ def test_slicing_and_indexing(arg):
cdef int[:] d = c[2, 1:2]
print b.shape[0], b.shape[1]
print b.strides[0], b.strides[1]
print_int_offsets(b.strides[0], b.strides[1])
cdef int i, j
for i in range(b.shape[0]):
......@@ -1230,3 +1318,15 @@ def test_slicing_and_indexing(arg):
print c[1, 1], c[2, 0]
print [d[i] for i in range(d.shape[0])]
@testcase
def test_oob():
"""
>>> test_oob()
Traceback (most recent call last):
...
IndexError: Index out of bounds (axis 1)
"""
cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9))
print a[:, 20]
tests/run/numpy_memoryview.pyx
View file @ b7e14b9a
......@@ -6,13 +6,15 @@ Test slicing for memoryviews and memoryviewslices
"""
cimport numpy as np
import numpy
import numpy as np
ctypedef np.int32_t dtype_t
def get_array():
# We need to type our array to get a __pyx_get_buffer() that typechecks
# for np.ndarray and calls __getbuffer__ in numpy.pxd
cdef np.ndarray[int, ndim=3] a
a = numpy.arange(8 * 14 * 11).reshape(8, 14, 11)
cdef np.ndarray[dtype_t, ndim=3] a
a = np.arange(8 * 14 * 11, dtype=np.int32).reshape(8, 14, 11)
return a
a = get_array()
......@@ -31,11 +33,11 @@ def test_partial_slicing(array):
"""
>>> test_partial_slicing(a)
"""
cdef int[:, :, :] a = array
cdef dtype_t[:, :, :] a = array
obj = array[4]
cdef int[:, :] b = a[4, :]
cdef int[:, :] c = a[4]
cdef dtype_t[:, :] b = a[4, :]
cdef dtype_t[:, :] c = a[4]
ae(b.shape[0], c.shape[0], obj.shape[0])
ae(b.shape[1], c.shape[1], obj.shape[1])
......@@ -46,15 +48,15 @@ def test_ellipsis(array):
"""
>>> test_ellipsis(a)
"""
cdef int[:, :, :] a = array
cdef dtype_t[:, :, :] a = array
cdef int[:, :] b = a[..., 4]
cdef dtype_t[:, :] b = a[..., 4]
b_obj = array[..., 4]
cdef int[:, :] c = a[4, ...]
cdef dtype_t[:, :] c = a[4, ...]
c_obj = array[4, ...]
cdef int[:, :] d = a[2:8, ..., 2]
cdef dtype_t[:, :] d = a[2:8, ..., 2]
d_obj = array[2:8, ..., 2]
ae(tuple([b.shape[i] for i in range(2)]), b_obj.shape)
......@@ -75,7 +77,7 @@ def test_ellipsis(array):
for j in range(d.shape[1]):
ae(d[i, j], d_obj[i, j])
cdef int[:] e = a[..., 5, 6]
cdef dtype_t[:] e = a[..., 5, 6]
e_obj = array[..., 5, 6]
ae(e.shape[0], e_obj.shape[0])
ae(e.strides[0], e_obj.strides[0])
......@@ -88,7 +90,7 @@ def test_partial_slicing_memoryview(array):
"""
>>> test_partial_slicing_memoryview(a)
"""
cdef int[:, :, :] _a = array
cdef dtype_t[:, :, :] _a = array
a = _a
obj = array[4]
......@@ -104,7 +106,7 @@ def test_ellipsis_memoryview(array):
"""
>>> test_ellipsis_memoryview(a)
"""
cdef int[:, :, :] _a = array
cdef dtype_t[:, :, :] _a = array
a = _a
b = a[..., 4]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7