Skip to content

C
cython
Commit 83e7a363 authored by Mark Florisson's avatar Mark Florisson
Browse files
Options

MemoryViewSlice indexing and object coercion + MemoryView indexing

parent aad49345
Show whitespace changes
Inline Side-by-side
Showing with 2119 additions and 570 deletions
Cython/Compiler/Buffer.py
View file @ 83e7a363
...@@ -215,6 +215,50 @@ class BufferEntry(object): ...@@ -215,6 +215,50 @@ class BufferEntry(object):
def _for_all_ndim(self, s): def _for_all_ndim(self, s):
return [s % (self.cname, i) for i in range(self.type.ndim)] return [s % (self.cname, i) for i in range(self.type.ndim)]
def generate_buffer_lookup_code(self, code, index_cnames):
# Create buffer lookup and return it
# This is done via utility macros/inline functions, which vary
# according to the access mode used.
params = []
nd = self.type.ndim
mode = self.type.mode
if mode == 'full':
for i, s, o in zip(index_cnames,
self.get_buf_stridevars(),
self.get_buf_suboffsetvars()):
params.append(i)
params.append(s)
params.append(o)
funcname = "__Pyx_BufPtrFull%dd" % nd
funcgen = buf_lookup_full_code
else:
if mode == 'strided':
funcname = "__Pyx_BufPtrStrided%dd" % nd
funcgen = buf_lookup_strided_code
elif mode == 'c':
funcname = "__Pyx_BufPtrCContig%dd" % nd
funcgen = buf_lookup_c_code
elif mode == 'fortran':
funcname = "__Pyx_BufPtrFortranContig%dd" % nd
funcgen = buf_lookup_fortran_code
else:
assert False
for i, s in zip(index_cnames, self.get_buf_stridevars()):
params.append(i)
params.append(s)
# Make sure the utility code is available
if funcname not in code.globalstate.utility_codes:
code.globalstate.utility_codes.add(funcname)
protocode = code.globalstate['utility_code_proto']
defcode = code.globalstate['utility_code_def']
funcgen(protocode, defcode, name=funcname, nd=nd)
buf_ptr_type_code = self.buf_ptr_type.declaration_code("")
ptrcode = "%s(%s, %s, %s)" % (funcname, buf_ptr_type_code, self.buf_ptr,
", ".join(params))
return ptrcode
def get_flags(buffer_aux, buffer_type): def get_flags(buffer_aux, buffer_type):
flags = 'PyBUF_FORMAT' flags = 'PyBUF_FORMAT'
...@@ -426,48 +470,7 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives, ...@@ -426,48 +470,7 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives,
if signed != 0: if signed != 0:
code.putln("if (%s < 0) %s += %s;" % (cname, cname, shape)) code.putln("if (%s < 0) %s += %s;" % (cname, cname, shape))
# Create buffer lookup and return it return entry.generate_buffer_lookup_code(code, index_cnames)
# This is done via utility macros/inline functions, which vary
# according to the access mode used.
params = []
nd = entry.type.ndim
mode = entry.type.mode
if mode == 'full':
for i, s, o in zip(index_cnames,
entry.get_buf_stridevars(),
entry.get_buf_suboffsetvars()):
params.append(i)
params.append(s)
params.append(o)
funcname = "__Pyx_BufPtrFull%dd" % nd
funcgen = buf_lookup_full_code
else:
if mode == 'strided':
funcname = "__Pyx_BufPtrStrided%dd" % nd
funcgen = buf_lookup_strided_code
elif mode == 'c':
funcname = "__Pyx_BufPtrCContig%dd" % nd
funcgen = buf_lookup_c_code
elif mode == 'fortran':
funcname = "__Pyx_BufPtrFortranContig%dd" % nd
funcgen = buf_lookup_fortran_code
else:
assert False
for i, s in zip(index_cnames, entry.get_buf_stridevars()):
params.append(i)
params.append(s)
# Make sure the utility code is available
if funcname not in code.globalstate.utility_codes:
code.globalstate.utility_codes.add(funcname)
protocode = code.globalstate['utility_code_proto']
defcode = code.globalstate['utility_code_def']
funcgen(protocode, defcode, name=funcname, nd=nd)
buf_ptr_type_code = entry.buf_ptr_type.declaration_code("")
ptrcode = "%s(%s, %s, %s)" % (funcname, buf_ptr_type_code, entry.buf_ptr,
", ".join(params))
return ptrcode
def use_bufstruct_declare_code(env): def use_bufstruct_declare_code(env):
......
Cython/Compiler/Code.py
View file @ 83e7a363
...@@ -23,7 +23,6 @@ import DebugFlags ...@@ -23,7 +23,6 @@ import DebugFlags
import Errors import Errors
from Cython import Tempita as tempita from Cython import Tempita as tempita
from Cython.Utils import none_or_sub
try: try:
from __builtin__ import basestring from __builtin__ import basestring
except ImportError: except ImportError:
...@@ -162,11 +161,11 @@ class UtilityCodeBase(object): ...@@ -162,11 +161,11 @@ class UtilityCodeBase(object):
kwargs['impl'] = impl kwargs['impl'] = impl
if 'name' not in kwargs: if 'name' not in kwargs:
if from_file:
kwargs['name'] = os.path.splitext(from_file)[0]
else:
kwargs['name'] = util_code_name kwargs['name'] = util_code_name
if 'file' not in kwargs and from_file:
kwargs['file'] = from_file
return cls(**kwargs) return cls(**kwargs)
load = classmethod(load) load = classmethod(load)
...@@ -188,13 +187,8 @@ class UtilityCodeBase(object): ...@@ -188,13 +187,8 @@ class UtilityCodeBase(object):
proto, impl = utilities[util_code_name] proto, impl = utilities[util_code_name]
if context is not None: if context is not None:
if '__name' not in context: proto = sub_tempita(proto, context, from_file, util_code_name)
context['__name'] = util_code_name impl = sub_tempita(impl, context, from_file, util_code_name)
if proto:
proto = tempita.sub(proto, **context)
if impl:
impl = tempita.sub(impl, **context)
if cls.is_cython_utility: if cls.is_cython_utility:
# Remember line numbers # Remember line numbers
...@@ -204,19 +198,56 @@ class UtilityCodeBase(object): ...@@ -204,19 +198,56 @@ class UtilityCodeBase(object):
load_as_string = classmethod(load_as_string) load_as_string = classmethod(load_as_string)
def none_or_sub(self, s, context, tempita):
"""
Format a string in this utility code with context. If None, do nothing.
"""
if s is None:
return None
if tempita:
return sub_tempita(s, context, self.file, self.name)
return s % context
def __str__(self): def __str__(self):
return "<%s(%s)" % (type(self).__name__, self.name) return "<%s(%s)" % (type(self).__name__, self.name)
def sub_tempita(s, context, file, name):
"Run tempita on string s with context context."
if not s:
return None
if file:
context['__name'] = "%s:%s" % (file, name)
elif name:
context['__name'] = name
return tempita.sub(s, **context)
class UtilityCode(UtilityCodeBase): class UtilityCode(UtilityCodeBase):
# Stores utility code to add during code generation. """
# Stores utility code to add during code generation.
# See GlobalState.put_utility_code.
# See GlobalState.put_utility_code.
# hashes/equals by instance
hashes/equals by instance
proto C prototypes
impl implemenation code
init code to call on module initialization
requires utility code dependencies
proto_block the place in the resulting file where the prototype should
end up
name name of the utility code (or None)
file filename of the utility code file this utility was loaded
from (or None)
"""
def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None, def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
proto_block='utility_code_proto', name=None): proto_block='utility_code_proto', name=None, file=None):
# proto_block: Which code block to dump prototype in. See GlobalState. # proto_block: Which code block to dump prototype in. See GlobalState.
self.proto = proto self.proto = proto
self.impl = impl self.impl = impl
...@@ -227,11 +258,13 @@ class UtilityCode(UtilityCodeBase): ...@@ -227,11 +258,13 @@ class UtilityCode(UtilityCodeBase):
self.specialize_list = [] self.specialize_list = []
self.proto_block = proto_block self.proto_block = proto_block
self.name = name self.name = name
self.file = file
def get_tree(self): def get_tree(self):
pass pass
def specialize(self, pyrex_type=None, **data):
def specialize(self, pyrex_type=None, tempita=False, **data):
# Dicts aren't hashable... # Dicts aren't hashable...
if pyrex_type is not None: if pyrex_type is not None:
data['type'] = pyrex_type.declaration_code('') data['type'] = pyrex_type.declaration_code('')
...@@ -244,12 +277,15 @@ class UtilityCode(UtilityCodeBase): ...@@ -244,12 +277,15 @@ class UtilityCode(UtilityCodeBase):
requires = None requires = None
else: else:
requires = [r.specialize(data) for r in self.requires] requires = [r.specialize(data) for r in self.requires]
s = self._cache[key] = UtilityCode( s = self._cache[key] = UtilityCode(
none_or_sub(self.proto, data), self.none_or_sub(self.proto, data, tempita),
none_or_sub(self.impl, data), self.none_or_sub(self.impl, data, tempita),
none_or_sub(self.init, data), self.none_or_sub(self.init, data, tempita),
none_or_sub(self.cleanup, data), self.none_or_sub(self.cleanup, data, tempita),
requires, self.proto_block) requires,
self.proto_block)
self.specialize_list.append(s) self.specialize_list.append(s)
return s return s
...@@ -275,6 +311,29 @@ class UtilityCode(UtilityCodeBase): ...@@ -275,6 +311,29 @@ class UtilityCode(UtilityCodeBase):
self.cleanup(writer, output.module_pos) self.cleanup(writer, output.module_pos)
class ContentHashingUtilityCode(UtilityCode):
"UtilityCode that hashes and compares based on self.proto and self.impl"
def __hash__(self):
return hash((self.proto, self.impl))
def __eq__(self, other):
return (self.proto, self.impl) == (other.proto, other.impl)
class LazyUtilityCode(UtilityCodeBase):
"""
Utility code that calls a callback with the root code writer when
available. Useful when you only have 'env' but not 'code'.
"""
def __init__(self, callback):
self.callback = callback
def put_code(self, globalstate):
utility = self.callback(globalstate.rootwriter)
globalstate.use_utility_code(utility)
class FunctionState(object): class FunctionState(object):
# return_label string function return point label # return_label string function return point label
...@@ -1538,6 +1597,15 @@ class CCodeWriter(object): ...@@ -1538,6 +1597,15 @@ class CCodeWriter(object):
for entry in entries: for entry in entries:
self.put_var_xdecref_clear(entry) self.put_var_xdecref_clear(entry)
def put_incref_memoryviewslice(self, slice_cname, have_gil=False):
self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))
def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False):
self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))
def put_xgiveref_memoryviewslice(self, slice_cname):
self.put_xgiveref("%s.memview" % slice_cname)
def put_init_to_py_none(self, cname, type, nanny=True): def put_init_to_py_none(self, cname, type, nanny=True):
from PyrexTypes import py_object_type, typecast from PyrexTypes import py_object_type, typecast
py_none = typecast(type, py_object_type, "Py_None") py_none = typecast(type, py_object_type, "Py_None")
......
Cython/Compiler/CythonScope.py
View file @ 83e7a363
...@@ -10,10 +10,12 @@ import MemoryView ...@@ -10,10 +10,12 @@ import MemoryView
class CythonScope(ModuleScope): class CythonScope(ModuleScope):
is_cython_builtin = 1 is_cython_builtin = 1
def __init__(self): def __init__(self, context):
ModuleScope.__init__(self, u'cython', None, None) ModuleScope.__init__(self, u'cython', None, None)
self.pxd_file_loaded = True self.pxd_file_loaded = True
self.populate_cython_scope() self.populate_cython_scope()
# The Main.Context object
self.context = context
def lookup_type(self, name): def lookup_type(self, name):
# This function should go away when types are all first-level objects. # This function should go away when types are all first-level objects.
...@@ -80,6 +82,10 @@ class CythonScope(ModuleScope): ...@@ -80,6 +82,10 @@ class CythonScope(ModuleScope):
# The view sub-scope # The view sub-scope
# #
self.viewscope = viewscope = ModuleScope(u'cython.view', self, None) self.viewscope = viewscope = ModuleScope(u'cython.view', self, None)
# Hacky monkey patch
self.viewscope.global_scope = self.global_scope
self.declare_module('view', viewscope, None) self.declare_module('view', viewscope, None)
viewscope.is_cython_builtin = True viewscope.is_cython_builtin = True
viewscope.pxd_file_loaded = True viewscope.pxd_file_loaded = True
...@@ -93,7 +99,7 @@ def create_cython_scope(context, create_testscope): ...@@ -93,7 +99,7 @@ def create_cython_scope(context, create_testscope):
# One could in fact probably make it a singleton, # One could in fact probably make it a singleton,
# but not sure yet whether any code mutates it (which would kill reusing # but not sure yet whether any code mutates it (which would kill reusing
# it across different contexts) # it across different contexts)
scope = CythonScope() scope = CythonScope(context)
if create_testscope: if create_testscope:
scope.test_cythonscope() scope.test_cythonscope()
...@@ -129,6 +135,12 @@ cython_test_extclass_utility_code = \ ...@@ -129,6 +135,12 @@ cython_test_extclass_utility_code = \
cythonview_testscope_utility_code = load_testscope_utility("View.TestScope") cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
view_utility_code = MemoryView.load_memview_cy_utility( view_utility_code = MemoryView.load_memview_cy_utility(
"View.MemoryView", requires=(Buffer.GetAndReleaseBufferUtilityCode(),)) "View.MemoryView", context=MemoryView.context,
requires=[Buffer.GetAndReleaseBufferUtilityCode(),
cython_array_utility_code = MemoryView.load_memview_cy_utility("CythonArray") MemoryView.memviewslice_declare_code],
)
cython_array_utility_code = MemoryView.load_memview_cy_utility(
"CythonArray",
context=MemoryView.context,
requires=[view_utility_code])
Cython/Compiler/ExprNodes.py
View file @ 83e7a363
...@@ -587,7 +587,12 @@ class ExprNode(Node): ...@@ -587,7 +587,12 @@ class ExprNode(Node):
if dst_type.is_memoryviewslice: if dst_type.is_memoryviewslice:
import MemoryView import MemoryView
if not src.type.is_memoryviewslice: if not src.type.is_memoryviewslice:
if src.type.is_pyobject:
src = CoerceToMemViewSliceNode(src, dst_type, env) src = CoerceToMemViewSliceNode(src, dst_type, env)
else:
error(self.pos,
"Cannot convert '%s' to memoryviewslice" %
(src_type,))
elif not MemoryView.src_conforms_to_dst(src.type, dst_type): elif not MemoryView.src_conforms_to_dst(src.type, dst_type):
error(self.pos, "Memoryview '%s' not conformable to memoryview '%s'." % error(self.pos, "Memoryview '%s' not conformable to memoryview '%s'." %
(src.type, dst_type)) (src.type, dst_type))
...@@ -1282,6 +1287,7 @@ class NameNode(AtomicExprNode): ...@@ -1282,6 +1287,7 @@ class NameNode(AtomicExprNode):
# cf_is_null boolean Is uninitialized before this node # cf_is_null boolean Is uninitialized before this node
# cf_maybe_null boolean Maybe uninitialized before this node # cf_maybe_null boolean Maybe uninitialized before this node
# allow_null boolean Don't raise UnboundLocalError # allow_null boolean Don't raise UnboundLocalError
# nogil boolean Whether it is used in a nogil context
is_name = True is_name = True
is_cython_module = False is_cython_module = False
...@@ -1293,6 +1299,7 @@ class NameNode(AtomicExprNode): ...@@ -1293,6 +1299,7 @@ class NameNode(AtomicExprNode):
cf_maybe_null = True cf_maybe_null = True
cf_is_null = False cf_is_null = False
allow_null = False allow_null = False
nogil = False
def create_analysed_rvalue(pos, env, entry): def create_analysed_rvalue(pos, env, entry):
node = NameNode(pos) node = NameNode(pos)
...@@ -1452,6 +1459,7 @@ class NameNode(AtomicExprNode): ...@@ -1452,6 +1459,7 @@ class NameNode(AtomicExprNode):
self.is_used_as_rvalue = 1 self.is_used_as_rvalue = 1
def nogil_check(self, env): def nogil_check(self, env):
self.nogil = True
if self.is_used_as_rvalue: if self.is_used_as_rvalue:
entry = self.entry entry = self.entry
if entry.is_builtin: if entry.is_builtin:
...@@ -1655,9 +1663,16 @@ class NameNode(AtomicExprNode): ...@@ -1655,9 +1663,16 @@ class NameNode(AtomicExprNode):
else: else:
if self.type.is_memoryviewslice: if self.type.is_memoryviewslice:
import MemoryView import MemoryView
MemoryView.gen_acquire_memoryviewslice(rhs, self.type, MemoryView.put_acquire_memoryviewslice(rhs, self.type,
self.entry.is_cglobal, self.result(), self.pos, code) self.entry.is_cglobal, self.result(), self.pos, code)
# self.generate_acquire_memoryviewslice(rhs, code)
if isinstance(rhs, CoerceToMemViewSliceNode):
# We had a new reference, the lhs now has another,
# dispose of ours.
# code.put_xdecref_memoryviewslice(rhs.result())
code.put_decref("%s.memview" % rhs.result(),
cython_memoryview_ptr_type,
nanny=False)
elif self.type.is_buffer: elif self.type.is_buffer:
# Generate code for doing the buffer release/acquisition. # Generate code for doing the buffer release/acquisition.
...@@ -1736,12 +1751,17 @@ class NameNode(AtomicExprNode): ...@@ -1736,12 +1751,17 @@ class NameNode(AtomicExprNode):
'__Pyx_DelAttrString(%s, "%s")' % ( '__Pyx_DelAttrString(%s, "%s")' % (
Naming.module_cname, Naming.module_cname,
self.entry.name)) self.entry.name))
elif self.entry.type.is_pyobject: elif self.entry.type.is_pyobject or self.entry.type.is_memoryviewslice:
if not self.cf_is_null: if not self.cf_is_null:
if self.cf_maybe_null: if self.cf_maybe_null:
code.put_error_if_unbound(self.pos, self.entry) code.put_error_if_unbound(self.pos, self.entry)
if self.entry.type.is_pyobject:
code.put_decref(self.result(), self.ctype()) code.put_decref(self.result(), self.ctype())
code.putln('%s = NULL;' % self.result()) code.putln('%s = NULL;' % self.result())
else:
code.put_xdecref_memoryviewslice(self.entry.cname,
have_gil=not self.nogil)
else: else:
error(self.pos, "Deletion of C names not supported") error(self.pos, "Deletion of C names not supported")
...@@ -2347,10 +2367,12 @@ class IndexNode(ExprNode): ...@@ -2347,10 +2367,12 @@ class IndexNode(ExprNode):
buffer_access = False buffer_access = False
memoryviewslice_access = False memoryviewslice_access = False
if (self.base.type.is_memoryviewslice and not self.indices and is_memslice = self.base.type.is_memoryviewslice
if (is_memslice and not self.indices and
isinstance(self.index, EllipsisNode)): isinstance(self.index, EllipsisNode)):
memoryviewslice_access = True memoryviewslice_access = True
elif self.base.type.is_buffer or self.base.type.is_memoryviewslice: elif self.base.type.is_buffer or is_memslice:
if self.indices: if self.indices:
indices = self.indices indices = self.indices
else: else:
...@@ -2358,6 +2380,7 @@ class IndexNode(ExprNode): ...@@ -2358,6 +2380,7 @@ class IndexNode(ExprNode):
indices = self.index.args indices = self.index.args
else: else:
indices = [self.index] indices = [self.index]
if len(indices) == self.base.type.ndim: if len(indices) == self.base.type.ndim:
buffer_access = True buffer_access = True
skip_child_analysis = True skip_child_analysis = True
...@@ -2365,15 +2388,10 @@ class IndexNode(ExprNode): ...@@ -2365,15 +2388,10 @@ class IndexNode(ExprNode):
x.analyse_types(env) x.analyse_types(env)
if not x.type.is_int: if not x.type.is_int:
buffer_access = False buffer_access = False
if buffer_access: if buffer_access:
assert hasattr(self.base, "entry") # Must be a NameNode-like node assert hasattr(self.base, "entry") # Must be a NameNode-like node
# if self.base.type.is_memoryviewslice:
# assert hasattr(self.base, "entry")
# if self.indices or not isinstance(self.index, EllipsisNode):
# error(self.pos, "Memoryviews currently support ellipsis indexing only.")
# else: memoryviewslice_access = True
# On cloning, indices is cloned. Otherwise, unpack index into indices # On cloning, indices is cloned. Otherwise, unpack index into indices
assert not (buffer_access and isinstance(self.index, CloneNode)) assert not (buffer_access and isinstance(self.index, CloneNode))
...@@ -2386,7 +2404,10 @@ class IndexNode(ExprNode): ...@@ -2386,7 +2404,10 @@ class IndexNode(ExprNode):
if getting and self.type.is_pyobject: if getting and self.type.is_pyobject:
self.is_temp = True self.is_temp = True
if setting:
if setting and self.base.type.is_memoryviewslice:
self.type.writable_needed = True
elif setting:
if not self.base.entry.type.writable: if not self.base.entry.type.writable:
error(self.pos, "Writing to readonly buffer") error(self.pos, "Writing to readonly buffer")
else: else:
...@@ -3990,8 +4011,8 @@ class AttributeNode(ExprNode): ...@@ -3990,8 +4011,8 @@ class AttributeNode(ExprNode):
import MemoryView import MemoryView
MemoryView.put_assign_to_memviewslice(select_code, rhs.result(), self.type, MemoryView.put_assign_to_memviewslice(select_code, rhs.result(), self.type,
pos=self.pos, code=code) pos=self.pos, code=code)
if rhs.is_temp: #if rhs.is_temp:
code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type) # code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type)
if not self.type.is_memoryviewslice: if not self.type.is_memoryviewslice:
code.putln( code.putln(
"%s = %s;" % ( "%s = %s;" % (
...@@ -7647,46 +7668,14 @@ class CoerceToMemViewSliceNode(CoercionNode): ...@@ -7647,46 +7668,14 @@ class CoerceToMemViewSliceNode(CoercionNode):
self.type = dst_type self.type = dst_type
self.env = env self.env = env
self.is_temp = 1 self.is_temp = 1
self.arg = arg
def generate_result_code(self, code): def generate_result_code(self, code):
import MemoryView, Buffer self.type.create_from_py_utility_code(self.env)
memviewobj = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True) code.putln("%s = %s(%s);" % (self.result(),
buf_flag = MemoryView.get_buf_flag(self.type.axes) self.type.from_py_function,
code.putln("%s = (PyObject *) __pyx_memoryview_new(%s, %s);" % self.arg.py_result()))
(memviewobj, self.arg.py_result(), buf_flag))
code.putln(code.error_goto_if_PyErr(self.pos))
ndim = len(self.type.axes)
spec_int_arr = code.funcstate.allocate_temp(
PyrexTypes.c_array_type(PyrexTypes.c_int_type, ndim),
manage_ref=False)
specs_code = MemoryView.specs_to_code(self.type.axes)
for idx, cspec in enumerate(specs_code):
code.putln("%s[%d] = %s;" % (spec_int_arr, idx, cspec))
code.globalstate.use_utility_code(Buffer.acquire_utility_code)
code.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
dtype_typeinfo = Buffer.get_type_information_cname(code, self.type.dtype)
MemoryView.put_init_entry(self.result(), code)
code.putln("{")
code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" %
self.type.dtype.struct_nesting_depth())
result = self.result()
if self.type.is_c_contig:
c_or_f_flag = "__Pyx_IS_C_CONTIG"
elif self.type.is_f_contig:
c_or_f_flag = "__Pyx_IS_F_CONTIG"
else:
c_or_f_flag = "0"
code.putln(code.error_goto_if("-1 == __Pyx_ValidateAndInit_memviewslice("
"(struct __pyx_memoryview_obj *) %(memviewobj)s,"
" %(spec_int_arr)s, %(c_or_f_flag)s, %(ndim)d,"
" &%(dtype_typeinfo)s, __pyx_stack, &%(result)s)" % locals(), self.pos))
code.putln("}")
code.put_gotref(
code.as_pyobject("%s.memview" % self.result(), cython_memoryview_ptr_type))
code.funcstate.release_temp(memviewobj)
code.funcstate.release_temp(spec_int_arr)
class CastNode(CoercionNode): class CastNode(CoercionNode):
# Wrap a node in a C type cast. # Wrap a node in a C type cast.
......
Cython/Compiler/Main.py
View file @ 83e7a363
...@@ -430,7 +430,7 @@ def create_default_resultobj(compilation_source, options): ...@@ -430,7 +430,7 @@ def create_default_resultobj(compilation_source, options):
def run_pipeline(source, options, full_module_name = None): def run_pipeline(source, options, full_module_name = None):
import Pipeline import Pipeline
# Set up context
context = options.create_context() context = options.create_context()
# Set up source object # Set up source object
...@@ -438,6 +438,7 @@ def run_pipeline(source, options, full_module_name = None): ...@@ -438,6 +438,7 @@ def run_pipeline(source, options, full_module_name = None):
abs_path = os.path.abspath(source) abs_path = os.path.abspath(source)
source_ext = os.path.splitext(source)[1] source_ext = os.path.splitext(source)[1]
full_module_name = full_module_name or context.extract_module_name(source, options) full_module_name = full_module_name or context.extract_module_name(source, options)
if options.relative_path_in_code_position_comments: if options.relative_path_in_code_position_comments:
rel_path = full_module_name.replace('.', os.sep) + source_ext rel_path = full_module_name.replace('.', os.sep) + source_ext
if not abs_path.endswith(rel_path): if not abs_path.endswith(rel_path):
...@@ -584,7 +585,8 @@ def compile_multiple(sources, options): ...@@ -584,7 +585,8 @@ def compile_multiple(sources, options):
a CompilationResultSet. Performs timestamp checking and/or recursion a CompilationResultSet. Performs timestamp checking and/or recursion
if these are specified in the options. if these are specified in the options.
""" """
context = options.create_context() # run_pipeline creates the context
# context = options.create_context()
sources = [os.path.abspath(source) for source in sources] sources = [os.path.abspath(source) for source in sources]
processed = set() processed = set()
results = CompilationResultSet() results = CompilationResultSet()
......
Cython/Compiler/MemoryView.py
View file @ 83e7a363
...@@ -46,17 +46,21 @@ _spec_to_const = { ...@@ -46,17 +46,21 @@ _spec_to_const = {
'follow' : MEMVIEW_FOLLOW, 'follow' : MEMVIEW_FOLLOW,
} }
_spec_to_abbrev = {
'direct' : 'd',
'ptr' : 'p',
'full' : 'f',
'contig' : 'c',
'strided' : 's',
'follow' : '_',
}
memview_name = u'memoryview' memview_name = u'memoryview'
memview_typeptr_cname = '__pyx_memoryview_type' memview_typeptr_cname = '__pyx_memoryview_type'
memview_objstruct_cname = '__pyx_memoryview_obj' memview_objstruct_cname = '__pyx_memoryview_obj'
memviewslice_cname = u'__Pyx_memviewslice' memviewslice_cname = u'__Pyx_memviewslice'
def specs_to_code(specs):
arr = []
for access, packing in specs:
arr.append("(%s | %s)" % (_spec_to_const[access],
_spec_to_const[packing]))
return arr
def put_init_entry(mv_cname, code): def put_init_entry(mv_cname, code):
code.putln("%s.data = NULL;" % mv_cname) code.putln("%s.data = NULL;" % mv_cname)
...@@ -70,7 +74,7 @@ def mangle_dtype_name(dtype): ...@@ -70,7 +74,7 @@ def mangle_dtype_name(dtype):
def axes_to_str(axes): def axes_to_str(axes):
return "".join([access[0].upper()+packing[0] for (access, packing) in axes]) return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code): def put_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code):
# import MemoryView # import MemoryView
assert rhs.type.is_memoryviewslice assert rhs.type.is_memoryviewslice
...@@ -80,32 +84,17 @@ def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_p ...@@ -80,32 +84,17 @@ def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_p
else: else:
rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False) rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False)
code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type))) code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type)))
code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
if not rhs.result_in_temp():
code.put_incref("%s.memview" % rhstmp, cython_memoryview_ptr_type)
if lhs_is_cglobal:
code.put_gotref("%s.memview" % lhs_result)
#XXX: this is here because self.lhs_of_first_assignment is not set correctly,
# once that is working this should take that flag into account.
# See NameNode.generate_assignment_code
code.put_xdecref("%s.memview" % lhs_result, cython_memoryview_ptr_type)
if lhs_is_cglobal:
code.put_giveref("%s.memview" % rhstmp)
code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
put_assign_to_memviewslice(lhs_result, rhstmp, lhs_type, put_assign_to_memviewslice(lhs_result, rhstmp, lhs_type,
lhs_pos, code=code) lhs_pos, code=code)
if rhs.result_in_temp() or not pretty_rhs:
code.putln("%s.memview = 0;" % rhstmp)
if not pretty_rhs: if not pretty_rhs:
code.funcstate.release_temp(rhstmp) code.funcstate.release_temp(rhstmp)
def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code): def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code):
code.put_xdecref_memoryviewslice(lhs_cname)
code.put_incref_memoryviewslice(rhs_cname)
code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname)) code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname))
code.putln("%s.data = %s.data;" % (lhs_cname, rhs_cname)) code.putln("%s.data = %s.data;" % (lhs_cname, rhs_cname))
...@@ -195,6 +184,45 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): ...@@ -195,6 +184,45 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
def get_buf_shapevars(self): def get_buf_shapevars(self):
return self._for_all_ndim("%s.shape[%d]") return self._for_all_ndim("%s.shape[%d]")
def generate_buffer_lookup_code(self, code, index_cnames):
bufp = self.buf_ptr
type_decl = self.type.dtype.declaration_code("")
for dim, (access, packing) in enumerate(self.type.axes):
shape = "%s.shape[%d]" % (self.cname, dim)
stride = "%s.strides[%d]" % (self.cname, dim)
suboffset = "%s.suboffsets[%d]" % (self.cname, dim)
index = index_cnames[dim]
if access == 'full' and packing in ('strided', 'follow'):
code.globalstate.use_utility_code(memviewslice_index_helpers)
bufp = ('__pyx_memviewslice_index_full(%s, %s, %s)' %
(bufp, index, stride, suboffset))
elif access == 'full' and packing == 'contig':
# We can skip stride multiplication with the cast
code.globalstate.use_utility_code(memviewslice_index_helpers)
bufp = '((char *) ((%s *) %s) + %s)' % (type_decl, bufp, index)
bufp = ('__pyx_memviewslice_index_full_contig(%s, %s)' %
(bufp, suboffset))
elif access == 'ptr' and packing in ('strided', 'follow'):
bufp = ("(*((char **) %s + %s * %s) + %s)" %
(bufp, index, stride, suboffset))
elif access == 'ptr' and packing == 'contig':
bufp = "(*((char **) %s) + %s)" % (bufp, suboffset)
elif access == 'direct' and packing in ('strided', 'follow'):
bufp = "(%s + %s * %s)" % (bufp, index, stride)
else:
assert (access, packing) == ('direct', 'contig'), (access, packing)
bufp = '((char *) (((%s *) %s) + %s))' % (type_decl, bufp, index)
bufp = '( /* dim=%d */ %s )' % (dim, bufp)
return "((%s *) %s)" % (type_decl, bufp)
def get_copy_func_name(to_memview): def get_copy_func_name(to_memview):
base = "__Pyx_BufferNew_%s_From_%s_%s" base = "__Pyx_BufferNew_%s_From_%s_%s"
...@@ -370,10 +398,10 @@ class CopyFuncUtilCode(object): ...@@ -370,10 +398,10 @@ class CopyFuncUtilCode(object):
if self.to_memview.is_c_contig: if self.to_memview.is_c_contig:
mode = 'c' mode = 'c'
contig_flag = 'PyBUF_C_CONTIGUOUS' contig_flag = memview_c_contiguous
elif self.to_memview.is_f_contig: elif self.to_memview.is_f_contig:
mode = 'fortran' mode = 'fortran'
contig_flag = "PyBUF_F_CONTIGUOUS" contig_flag = memview_f_contiguous
context = dict( context = dict(
copy_name=self.copy_func_name, copy_name=self.copy_func_name,
...@@ -735,11 +763,13 @@ class MemoryViewSliceTransform(CythonTransform): ...@@ -735,11 +763,13 @@ class MemoryViewSliceTransform(CythonTransform):
return node return node
def load_memview_cy_utility(util_code_name, **kwargs): def load_memview_cy_utility(util_code_name, context=None, **kwargs):
return CythonUtilityCode.load(util_code_name, "MemoryView.pyx", **kwargs) return CythonUtilityCode.load(util_code_name, "MemoryView.pyx",
context=context, **kwargs)
def load_memview_c_utility(util_code_name, **kwargs): def load_memview_c_utility(util_code_name, context=None, **kwargs):
return UtilityCode.load(util_code_name, "MemoryView_C.c", **kwargs) return UtilityCode.load(util_code_name, "MemoryView_C.c",
context=context, **kwargs)
context = { context = {
'memview_struct_name': memview_objstruct_cname, 'memview_struct_name': memview_objstruct_cname,
...@@ -753,6 +783,8 @@ memviewslice_declare_code = load_memview_c_utility( ...@@ -753,6 +783,8 @@ memviewslice_declare_code = load_memview_c_utility(
memviewslice_init_code = load_memview_c_utility( memviewslice_init_code = load_memview_c_utility(
"MemviewSliceInit", "MemviewSliceInit",
context={'BUF_MAX_NDIMS': Options.buffer_max_dims}, context=dict(context, BUF_MAX_NDIMS=Options.buffer_max_dims),
requires=[memviewslice_declare_code], requires=[memviewslice_declare_code],
) )
memviewslice_index_helpers = load_memview_c_utility("MemviewSliceIndex")
\ No newline at end of file
Cython/Compiler/ModuleNode.py
View file @ 83e7a363
...@@ -79,6 +79,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -79,6 +79,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
scope.python_include_files) scope.python_include_files)
if merge_scope: if merge_scope:
# Ensure that we don't generate import code for these entries!
for entry in scope.c_class_entries:
entry.type.module_name = self.full_module_name
self.scope.merge_in(scope) self.scope.merge_in(scope)
def analyse_declarations(self, env): def analyse_declarations(self, env):
...@@ -345,7 +349,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -345,7 +349,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
self.generate_declarations_for_modules(env, modules, globalstate) self.generate_declarations_for_modules(env, modules, globalstate)
h_code.write('\n') h_code.write('\n')
for utilcode in env.utility_code_list: for utilcode in env.utility_code_list[:]:
globalstate.use_utility_code(utilcode) globalstate.use_utility_code(utilcode)
globalstate.finalize_main_c_code() globalstate.finalize_main_c_code()
...@@ -2255,7 +2259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -2255,7 +2259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
# Generate type import code for extern extension types # Generate type import code for extern extension types
# and type ready code for non-extern ones. # and type ready code for non-extern ones.
for entry in env.c_class_entries: for entry in env.c_class_entries:
if entry.visibility == 'extern': if entry.visibility == 'extern' and not entry.utility_code_definition:
self.generate_type_import_code(env, entry.type, entry.pos, code) self.generate_type_import_code(env, entry.type, entry.pos, code)
else: else:
self.generate_base_type_import_code(env, entry, code) self.generate_base_type_import_code(env, entry, code)
...@@ -2265,8 +2269,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -2265,8 +2269,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
def generate_base_type_import_code(self, env, entry, code): def generate_base_type_import_code(self, env, entry, code):
base_type = entry.type.base_type base_type = entry.type.base_type
if base_type and base_type.module_name != env.qualified_name \ if (base_type and base_type.module_name != env.qualified_name and not
and not base_type.is_builtin_type: base_type.is_builtin_type and not entry.utility_code_definition):
self.generate_type_import_code(env, base_type, self.pos, code) self.generate_type_import_code(env, base_type, self.pos, code)
def use_type_import_utility_code(self, env): def use_type_import_utility_code(self, env):
......
Cython/Compiler/Nodes.py
View file @ 83e7a363
...@@ -1344,17 +1344,19 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1344,17 +1344,19 @@ class FuncDefNode(StatNode, BlockNode):
if not entry.in_closure: if not entry.in_closure:
code.put_var_declaration(entry) code.put_var_declaration(entry)
# Initialize the return variable __pyx_r
init = "" init = ""
if not self.return_type.is_void: if not self.return_type.is_void:
if self.return_type.is_pyobject: if self.return_type.is_pyobject:
init = " = NULL" init = " = NULL"
elif self.return_type.is_memoryviewslice:
init = "= {0, 0}"
code.putln( code.putln(
"%s%s;" % "%s%s;" %
(self.return_type.declaration_code(Naming.retval_cname), (self.return_type.declaration_code(Naming.retval_cname),
init)) init))
if self.return_type.is_memoryviewslice:
import MemoryView
MemoryView.put_init_entry(Naming.retval_cname, code)
tempvardecl_code = code.insertion_point() tempvardecl_code = code.insertion_point()
self.generate_keyword_list(code) self.generate_keyword_list(code)
...@@ -1431,15 +1433,18 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1431,15 +1433,18 @@ class FuncDefNode(StatNode, BlockNode):
if (acquire_gil or entry.assignments) and not entry.in_closure: if (acquire_gil or entry.assignments) and not entry.in_closure:
code.put_var_incref(entry) code.put_var_incref(entry)
if entry.type.is_memoryviewslice: if entry.type.is_memoryviewslice:
code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type) code.put_incref_memoryviewslice(entry.cname,
have_gil=not lenv.nogil)
#code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
# ----- Initialise local buffer auxiliary variables # ----- Initialise local buffer auxiliary variables
for entry in lenv.var_entries + lenv.arg_entries: for entry in lenv.var_entries + lenv.arg_entries:
if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used: if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
Buffer.put_init_vars(entry, code) Buffer.put_init_vars(entry, code)
# ----- Initialise local memoryviewslices # ----- Initialise local memoryviewslices
for entry in lenv.var_entries: for entry in lenv.var_entries:
if entry.type.is_memoryviewslice: if entry.visibility == "private" and not entry.used:
MemoryView.put_init_entry(entry.cname, code) continue
# ----- Check and convert arguments # ----- Check and convert arguments
self.generate_argument_type_tests(code) self.generate_argument_type_tests(code)
# ----- Acquire buffer arguments # ----- Acquire buffer arguments
...@@ -1544,7 +1549,8 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1544,7 +1549,8 @@ class FuncDefNode(StatNode, BlockNode):
if not entry.used or entry.in_closure: if not entry.used or entry.in_closure:
continue continue
if entry.type.is_memoryviewslice: if entry.type.is_memoryviewslice:
code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type) #code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
code.put_xdecref_memoryviewslice(entry.cname)
if entry.type.is_pyobject: if entry.type.is_pyobject:
code.put_var_decref(entry) code.put_var_decref(entry)
...@@ -1554,7 +1560,8 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1554,7 +1560,8 @@ class FuncDefNode(StatNode, BlockNode):
if (acquire_gil or entry.assignments) and not entry.in_closure: if (acquire_gil or entry.assignments) and not entry.in_closure:
code.put_var_decref(entry) code.put_var_decref(entry)
if entry.type.is_memoryviewslice: if entry.type.is_memoryviewslice:
code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type) code.put_xdecref_memoryviewslice(entry.cname)
#code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
if self.needs_closure: if self.needs_closure:
code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type) code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
...@@ -1567,8 +1574,9 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1567,8 +1574,9 @@ class FuncDefNode(StatNode, BlockNode):
err_val = default_retval err_val = default_retval
if self.return_type.is_pyobject: if self.return_type.is_pyobject:
code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname)) code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
elif self.return_type.is_memoryviewslice: #elif self.return_type.is_memoryviewslice:
code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type)) # code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type))
# code.put_xgiveref_memoryviewslice(Naming.retval_cname)
if self.entry.is_special and self.entry.name == "__hash__": if self.entry.is_special and self.entry.name == "__hash__":
# Returning -1 for __hash__ is supposed to signal an error # Returning -1 for __hash__ is supposed to signal an error
...@@ -4153,7 +4161,8 @@ class DelStatNode(StatNode): ...@@ -4153,7 +4161,8 @@ class DelStatNode(StatNode):
def analyse_expressions(self, env): def analyse_expressions(self, env):
for arg in self.args: for arg in self.args:
arg.analyse_target_expression(env, None) arg.analyse_target_expression(env, None)
if arg.type.is_pyobject: if arg.type.is_pyobject or (arg.is_name and
arg.type.is_memoryviewslice):
pass pass
elif arg.type.is_ptr and arg.type.base_type.is_cpp_class: elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
self.cpp_check(env) self.cpp_check(env)
...@@ -4172,7 +4181,7 @@ class DelStatNode(StatNode): ...@@ -4172,7 +4181,7 @@ class DelStatNode(StatNode):
def generate_execution_code(self, code): def generate_execution_code(self, code):
for arg in self.args: for arg in self.args:
if arg.type.is_pyobject: if arg.type.is_pyobject or arg.type.is_memoryviewslice:
arg.generate_deletion_code(code) arg.generate_deletion_code(code)
elif arg.type.is_ptr and arg.type.base_type.is_cpp_class: elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
arg.generate_result_code(code) arg.generate_result_code(code)
...@@ -4274,14 +4283,15 @@ class ReturnStatNode(StatNode): ...@@ -4274,14 +4283,15 @@ class ReturnStatNode(StatNode):
code.put_xdecref(Naming.retval_cname, code.put_xdecref(Naming.retval_cname,
self.return_type) self.return_type)
elif self.return_type.is_memoryviewslice: elif self.return_type.is_memoryviewslice:
code.put_xdecref("%s.memview" % Naming.retval_cname, code.put_xdecref_memoryviewslice(Naming.retval_cname)
self.return_type) #code.put_xdecref("%s.memview" % Naming.retval_cname,
# self.return_type)
if self.value: if self.value:
self.value.generate_evaluation_code(code) self.value.generate_evaluation_code(code)
if self.return_type.is_memoryviewslice: if self.return_type.is_memoryviewslice:
import MemoryView import MemoryView
MemoryView.gen_acquire_memoryviewslice(self.value, self.return_type, MemoryView.put_acquire_memoryviewslice(self.value, self.return_type,
False, Naming.retval_cname, None, code) False, Naming.retval_cname, None, code)
else: else:
self.value.make_owned_reference(code) self.value.make_owned_reference(code)
......
Cython/Compiler/PyrexTypes.py
View file @ 83e7a363
...@@ -2,12 +2,14 @@ ...@@ -2,12 +2,14 @@
# Cython/Python language types # Cython/Python language types
# #
from Code import UtilityCode from Code import UtilityCode, LazyUtilityCode
import StringEncoding import StringEncoding
import Naming import Naming
import copy import copy
from Errors import error from Errors import error
import cython
class BaseType(object): class BaseType(object):
# #
# Base class for all Cython types including pseudo-types. # Base class for all Cython types including pseudo-types.
...@@ -331,6 +333,15 @@ class MemoryViewSliceType(PyrexType): ...@@ -331,6 +333,15 @@ class MemoryViewSliceType(PyrexType):
has_attributes = 1 has_attributes = 1
scope = None scope = None
# These are specialcased in Defnode
from_py_function = None
to_py_function = None
exception_value = None
exception_check = None
utility_counter = 0
def __init__(self, base_dtype, axes): def __init__(self, base_dtype, axes):
''' '''
MemoryViewSliceType(base, axes) MemoryViewSliceType(base, axes)
...@@ -375,6 +386,7 @@ class MemoryViewSliceType(PyrexType): ...@@ -375,6 +386,7 @@ class MemoryViewSliceType(PyrexType):
assert not (self.is_c_contig and self.is_f_contig) assert not (self.is_c_contig and self.is_f_contig)
self.mode = MemoryView.get_mode(axes) self.mode = MemoryView.get_mode(axes)
self.writable_needed = False
def same_as_resolved_type(self, other_type): def same_as_resolved_type(self, other_type):
return ((other_type.is_memoryviewslice and return ((other_type.is_memoryviewslice and
...@@ -495,11 +507,68 @@ class MemoryViewSliceType(PyrexType): ...@@ -495,11 +507,68 @@ class MemoryViewSliceType(PyrexType):
def global_init_code(self, entry, code): def global_init_code(self, entry, code):
code.putln("%s.data = NULL;" % entry.cname) code.putln("%s.data = NULL;" % entry.cname)
code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False) code.putln("%s.memview = NULL;" % entry.cname)
#code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False)
def check_for_null_code(self, cname): def check_for_null_code(self, cname):
return cname + '.memview' return cname + '.memview'
def create_from_py_utility_code(self, env):
import MemoryView, Buffer, Code
# We don't have 'code', so use a LazyUtilityCode with a callback.
def lazy_utility_callback(code):
context['dtype_typeinfo'] = Buffer.get_type_information_cname(
code, self.dtype)
return Code.ContentHashingUtilityCode.load(
"ObjectToMemviewSlice", "MemoryView_C.c", context)
env.use_utility_code(Buffer.acquire_utility_code)
env.use_utility_code(MemoryView.memviewslice_init_code)
env.use_utility_code(LazyUtilityCode(lazy_utility_callback))
if self.is_c_contig:
c_or_f_flag = "__Pyx_IS_C_CONTIG"
elif self.is_f_contig:
c_or_f_flag = "__Pyx_IS_F_CONTIG"
else:
c_or_f_flag = "0"
# specializing through UtilityCode.specialize is not so useful as
# specialize on too many things to include in the function name
funcname = "__Pyx_PyObject_to_MemoryviewSlice_%d" % self.utility_counter
context = dict(
MemoryView.context,
buf_flag = MemoryView.get_buf_flag(self.axes),
ndim = self.ndim,
axes_specs = ', '.join(self.axes_specs_to_code()),
dtype_typedecl = self.dtype.declaration_code(""),
struct_nesting_depth = self.dtype.struct_nesting_depth(),
c_or_f_flag = c_or_f_flag,
funcname = funcname,
)
self.from_py_function = funcname
MemoryViewSliceType.utility_counter += 1
return True
def axes_specs_to_code(self):
"Return a list of code constants for each axis"
import MemoryView
d = MemoryView._spec_to_const
return ["(%s | %s)" % (d[a], d[p]) for a, p in self.axes]
def axes_specs_to_name(self):
"Return an abbreviated name for our axes"
import MemoryView
d = MemoryView._spec_to_abbrev
return "".join(["%s%s" % (d[a], d[p]) for a, p in self.axes])
def error_condition(self, result_code):
return "!%s.memview" % result_code
class BufferType(BaseType): class BufferType(BaseType):
# #
...@@ -2698,7 +2767,7 @@ cython_memoryview_type = CStructOrUnionType("__pyx_memoryview_obj", "struct", ...@@ -2698,7 +2767,7 @@ cython_memoryview_type = CStructOrUnionType("__pyx_memoryview_obj", "struct",
cython_memoryview_ptr_type = CPtrType(cython_memoryview_type) cython_memoryview_ptr_type = CPtrType(cython_memoryview_type)
memoryviewslice_type = CStructOrUnionType("__Pyx_memviewslice", "struct", memoryviewslice_type = CStructOrUnionType("memoryviewslice", "struct",
None, 1, "__Pyx_memviewslice") None, 1, "__Pyx_memviewslice")
error_type = ErrorType() error_type = ErrorType()
......
Cython/Compiler/Symtab.py
View file @ 83e7a363
...@@ -382,6 +382,10 @@ class Scope(object): ...@@ -382,6 +382,10 @@ class Scope(object):
# entries[name] = entry # entries[name] = entry
if not shadow: if not shadow:
entries[name] = entry entries[name] = entry
if type.is_memoryviewslice:
entry.init = "{ 0, 0 }"
entry.scope = self entry.scope = self
entry.visibility = visibility entry.visibility = visibility
return entry return entry
......
Cython/Compiler/UtilityCode.py
View file @ 83e7a363
...@@ -63,7 +63,8 @@ class CythonUtilityCode(Code.UtilityCodeBase): ...@@ -63,7 +63,8 @@ class CythonUtilityCode(Code.UtilityCodeBase):
is_cython_utility = True is_cython_utility = True
def __init__(self, impl, name="__pyxutil", prefix="", requires=None): def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
file=None):
# 1) We need to delay the parsing/processing, so that all modules can be # 1) We need to delay the parsing/processing, so that all modules can be
# imported without import loops # imported without import loops
# 2) The same utility code object can be used for multiple source files; # 2) The same utility code object can be used for multiple source files;
...@@ -72,6 +73,7 @@ class CythonUtilityCode(Code.UtilityCodeBase): ...@@ -72,6 +73,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
# Hence, delay any processing until later. # Hence, delay any processing until later.
self.impl = impl self.impl = impl
self.name = name self.name = name
self.file = file
self.prefix = prefix self.prefix = prefix
self.requires = requires or [] self.requires = requires or []
...@@ -113,10 +115,11 @@ class CythonUtilityCode(Code.UtilityCodeBase): ...@@ -113,10 +115,11 @@ class CythonUtilityCode(Code.UtilityCodeBase):
def put_code(self, output): def put_code(self, output):
pass pass
def declare_in_scope(self, dest_scope, used=False): def declare_in_scope(self, dest_scope, used=False, modname=None):
""" """
Declare all entries from the utility code in dest_scope. Code will only Declare all entries from the utility code in dest_scope. Code will only
be included for used entries. be included for used entries. If module_name is given, declare the
type entries with that name.
""" """
tree = self.get_tree(entries_only=True) tree = self.get_tree(entries_only=True)
...@@ -130,6 +133,10 @@ class CythonUtilityCode(Code.UtilityCodeBase): ...@@ -130,6 +133,10 @@ class CythonUtilityCode(Code.UtilityCodeBase):
entry.utility_code_definition = self entry.utility_code_definition = self
entry.used = used entry.used = used
if modname and entry.type.is_extension_type:
entry.qualified_name = modname
entry.type.module_name = modname
dest_scope.merge_in(tree.scope, merge_unused=True) dest_scope.merge_in(tree.scope, merge_unused=True)
tree.scope = dest_scope tree.scope = dest_scope
......
Cython/Utility/MemoryView.pyx
View file @ 83e7a363
...@@ -10,7 +10,10 @@ cdef extern from "Python.h": ...@@ -10,7 +10,10 @@ cdef extern from "Python.h":
PyBUF_C_CONTIGUOUS, PyBUF_C_CONTIGUOUS,
PyBUF_F_CONTIGUOUS, PyBUF_F_CONTIGUOUS,
PyBUF_ANY_CONTIGUOUS PyBUF_ANY_CONTIGUOUS
PyBUF_FORMAT
cdef extern from *:
object __pyx_memoryview_new(object obj, int flags)
@cname("__pyx_array") @cname("__pyx_array")
cdef class array: cdef class array:
...@@ -105,7 +108,12 @@ cdef class array: ...@@ -105,7 +108,12 @@ cdef class array:
info.strides = self.strides info.strides = self.strides
info.suboffsets = NULL info.suboffsets = NULL
info.itemsize = self.itemsize info.itemsize = self.itemsize
if flags & PyBUF_FORMAT:
info.format = self.format info.format = self.format
else:
info.format = NULL
# we do not need to call releasebuffer # we do not need to call releasebuffer
info.obj = None info.obj = None
...@@ -130,6 +138,15 @@ cdef class array: ...@@ -130,6 +138,15 @@ cdef class array:
self.format = NULL self.format = NULL
self.itemsize = 0 self.itemsize = 0
def __getitem__(self, index):
view = __pyx_memoryview_new(self, PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT)
return view[index]
def __setitem__(self, index, value):
view = __pyx_memoryview_new(self, PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT)
view[index] = value
@cname("__pyx_array_new") @cname("__pyx_array_new")
cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode): cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode):
return array(shape, itemsize, format, mode.decode('ASCII')) return array(shape, itemsize, format, mode.decode('ASCII'))
...@@ -137,8 +154,10 @@ cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char * ...@@ -137,8 +154,10 @@ cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *
########## View.MemoryView ########## ########## View.MemoryView ##########
# from cpython cimport ... # from cpython cimport ...
cdef extern from "pythread.h": cdef extern from "Python.h":
int PyIndex_Check(object)
cdef extern from "pythread.h":
ctypedef void *PyThread_type_lock ctypedef void *PyThread_type_lock
PyThread_type_lock PyThread_allocate_lock() PyThread_type_lock PyThread_allocate_lock()
...@@ -150,6 +169,12 @@ cdef extern from *: ...@@ -150,6 +169,12 @@ cdef extern from *:
int __Pyx_GetBuffer(object, Py_buffer *, int) int __Pyx_GetBuffer(object, Py_buffer *, int)
void __Pyx_ReleaseBuffer(Py_buffer *) void __Pyx_ReleaseBuffer(Py_buffer *)
ctypedef struct {{memviewslice_name}}:
char *data
Py_ssize_t shape[{{max_dims}}]
Py_ssize_t strides[{{max_dims}}]
Py_ssize_t suboffsets[{{max_dims}}]
@cname('__pyx_MemviewEnum') @cname('__pyx_MemviewEnum')
cdef class Enum(object): cdef class Enum(object):
...@@ -173,23 +198,143 @@ cdef class memoryview(object): ...@@ -173,23 +198,143 @@ cdef class memoryview(object):
cdef object obj cdef object obj
cdef Py_buffer view cdef Py_buffer view
cdef PyThread_type_lock acqcnt_lock cdef PyThread_type_lock lock
cdef int acquisition_count cdef int acquisition_count
def __cinit__(memoryview self, object obj, int flags): def __cinit__(memoryview self, object obj, int flags):
self.obj = obj self.obj = obj
#self.acqcnt_lock = PyThread_allocate_lock()
#if self.acqcnt_lock == NULL:
# raise MemoryError
__Pyx_GetBuffer(obj, &self.view, flags) __Pyx_GetBuffer(obj, &self.view, flags)
self.lock = PyThread_allocate_lock()
if self.lock == NULL:
raise MemoryError
def __dealloc__(memoryview self): def __dealloc__(memoryview self):
#PyThread_free_lock(self.acqcnt_lock)
self.obj = None
__Pyx_ReleaseBuffer(&self.view) __Pyx_ReleaseBuffer(&self.view)
PyThread_free_lock(self.lock)
@cname('__pyx_memoryview_getitem')
def __getitem__(memoryview self, object index):
# cdef Py_ssize_t idx
cdef char *itemp = <char *> self.view.buf
cdef bytes bytesitem
cdef str fmt = self.view.format
import struct
try:
itemsize = struct.calcsize(fmt)
except struct.error:
raise TypeError("Unsupported format: %r" % fmt)
if index is Ellipsis:
return self
elif isinstance(index, slice):
if index == slice(None):
return self
raise NotImplementedError
else:
if not isinstance(index, tuple):
index = (index,)
tup = _unellipsify(index, self.view.ndim)
if len(tup) != self.view.ndim:
raise NotImplementedError(
"Expected %d indices (got %d)" %
(self.view.ndim, len(tup)))
for dim, idx in enumerate(tup):
_check_index(idx)
itemp = pybuffer_index(&self.view, itemp, idx, dim + 1)
# Do a manual and complete check here instead of this easy hack
bytesitem = itemp[:self.view.itemsize]
return struct.unpack(fmt, bytesitem)
@cname('__pyx_memoryviewslice')
cdef class _memoryviewslice(memoryview):
"Internal class for passing memory view slices to Python"
# We need this to keep our shape/strides/suboffset pointers valid
cdef {{memviewslice_name}} from_slice
# Restore the original Py_buffer before releasing
cdef Py_buffer orig_view
def __cinit__(self, object obj, int flags):
self.orig_view = self.view
def __dealloc__(self):
self.view = self.orig_view
@cname('__pyx_memoryview_new') @cname('__pyx_memoryview_new')
cdef memoryview memoryview_cwrapper(object o, int flags): cdef memoryview_cwrapper(object o, int flags):
return memoryview(o, flags) return memoryview(o, flags)
@cname('__pyx_memoryview_fromslice')
cdef memoryview memoryview_from_memview_cwrapper(memoryview m, int flags,
int new_ndim, {{memviewslice_name}} *memviewslice):
cdef _memoryviewslice result = _memoryviewslice(m.obj, flags)
result.from_slice = memviewslice[0]
result.view.shape = <Py_ssize_t *> (&result.from_slice.shape + new_ndim)
result.view.strides = <Py_ssize_t *> (&result.from_slice.strides + new_ndim)
result.view.suboffsets = <Py_ssize_t *> (&result.from_slice.suboffsets + new_ndim)
result.view.ndim = new_ndim
return result
cdef _check_index(index):
if not PyIndex_Check(index):
raise TypeError("Cannot index with %s" % type(index))
cdef tuple _unellipsify(tuple tup, int ndim):
if Ellipsis in tup:
result = []
for idx, item in enumerate(tup):
if item is Ellipsis:
result.extend([slice(None)] * (ndim - len(tup) + 1))
result.extend(tup[idx + 1:])
break
result.append(item)
return tuple(result)
return tup
@cname('__pyx_pybuffer_index')
cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, int dim) except NULL:
cdef Py_ssize_t shape, stride, suboffset = -1
cdef Py_ssize_t itemsize = view.itemsize
cdef char *resultp
if view.ndim == 0:
shape = view.len / itemsize
stride = itemsize
else:
shape = view.shape[dim]
stride = view.strides[dim]
if view.suboffsets != NULL:
suboffset = view.suboffsets[dim]
if index < 0:
index += view.shape[dim]
if index < 0:
raise IndexError("Out of bounds in dimension %d" % dim)
if index > shape:
raise IndexError("Out of bounds in dimension %d" % dim)
resultp = bufp + index * stride
if suboffset >= 0:
resultp = (<char **> resultp)[0] + suboffset
return resultp
Cython/Utility/MemoryView_C.c
View file @ 83e7a363
...@@ -12,6 +12,9 @@ typedef struct { ...@@ -12,6 +12,9 @@ typedef struct {
Py_ssize_t suboffsets[{{max_dims}}]; Py_ssize_t suboffsets[{{max_dims}}];
} {{memviewslice_name}}; } {{memviewslice_name}};
/////////////// ObjectToMemviewSlice.proto ///////////////
{{# __Pyx_PyObject_to_MemoryviewSlice_<count> }}
static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *);
////////// MemviewSliceInit.proto ////////// ////////// MemviewSliceInit.proto //////////
...@@ -27,6 +30,7 @@ typedef struct { ...@@ -27,6 +30,7 @@ typedef struct {
#define __Pyx_IS_C_CONTIG 1 #define __Pyx_IS_C_CONTIG 1
#define __Pyx_IS_F_CONTIG 2 #define __Pyx_IS_F_CONTIG 2
/* #define __PYX_MEMSLICE_GETDATA(SLICE) ((char *) SLICE->memview->view->buf) */ /* #define __PYX_MEMSLICE_GETDATA(SLICE) ((char *) SLICE->memview->view->buf) */
static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview, static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview,
...@@ -38,6 +42,49 @@ static int __Pyx_init_memviewslice( ...@@ -38,6 +42,49 @@ static int __Pyx_init_memviewslice(
int ndim, int ndim,
__Pyx_memviewslice *memviewslice); __Pyx_memviewslice *memviewslice);
#define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__)
#define __PYX_XDEC_MEMVIEW(slice, have_gil) __Pyx_XDEC_MEMVIEW(slice, have_gil, __LINE__)
static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int);
static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int);
/////////////// MemviewSliceIndex.proto ///////////////
static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset);
static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset);
/////////////// ObjectToMemviewSlice ///////////////
{{#__Pyx_PyObject_to_MemoryviewSlice_<count>}}
static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
{{memviewslice_name}} result;
result.memview = NULL;
result.data = NULL;
struct __pyx_memoryview_obj *memview = \
(struct __pyx_memoryview_obj *) __pyx_memoryview_new(obj, {{buf_flag}});
__Pyx_BufFmt_StackElem stack[{{struct_nesting_depth}}];
int axes_specs[] = { {{axes_specs}} };
int retcode;
if (unlikely(!memview))
goto __pyx_fail;
retcode = __Pyx_ValidateAndInit_memviewslice(memview, axes_specs,
{{c_or_f_flag}}, {{ndim}}, &{{dtype_typeinfo}}, stack, &result);
if (unlikely(retcode == -1))
goto __pyx_fail;
memview->acquisition_count = 1;
return result;
__pyx_fail:
Py_XDECREF(memview);
result.memview = NULL;
result.data = NULL;
return result;
}
////////// MemviewSliceInit ////////// ////////// MemviewSliceInit //////////
static int __Pyx_ValidateAndInit_memviewslice( static int __Pyx_ValidateAndInit_memviewslice(
...@@ -203,8 +250,6 @@ static int __Pyx_init_memviewslice( ...@@ -203,8 +250,6 @@ static int __Pyx_init_memviewslice(
} }
} }
__Pyx_INCREF((PyObject *)memview);
__Pyx_GIVEREF((PyObject *)memview);
memviewslice->memview = memview; memviewslice->memview = memview;
memviewslice->data = (char *)buf->buf; memviewslice->data = (char *)buf->buf;
retval = 0; retval = 0;
...@@ -220,6 +265,62 @@ no_fail: ...@@ -220,6 +265,62 @@ no_fail:
return retval; return retval;
} }
static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
int have_gil, int lineno) {
int first_time;
struct {{memview_struct_name}} *memview = memslice->memview;
if (!memview) {
char msg[50];
snprintf(msg, 50, "memoryslice is not initialized (line %d)", lineno);
Py_FatalError(msg);
}
PyThread_acquire_lock(memview->lock, 1);
first_time = (memview->acquisition_count++ == 0);
PyThread_release_lock(memview->lock);
/* printf("INCREF %d: acquisition_count=%d, refcount=%d\n", lineno,
memview->acquisition_count, memview->ob_refcnt); */
if (first_time) {
if (have_gil) {
Py_INCREF((PyObject *) memview);
} else {
PyGILState_STATE _gilstate = PyGILState_Ensure();
Py_INCREF((PyObject *) memview);
PyGILState_Release(_gilstate);
}
}
}
static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
int have_gil, int lineno) {
int last_time;
struct {{memview_struct_name}} *memview = memslice->memview;
if (!memview) {
return;
}
PyThread_acquire_lock(memview->lock, 1);
last_time = (memview->acquisition_count-- == 1);
PyThread_release_lock(memview->lock);
/* printf("DECREF %d: acquisition_count=%d, refcount=%d\n", lineno,
memview->acquisition_count, memview->ob_refcnt); */
if (last_time) {
if (have_gil) {
Py_CLEAR(memview);
} else {
PyGILState_STATE _gilstate = PyGILState_Ensure();
Py_CLEAR(memview);
PyGILState_Release(_gilstate);
}
memslice->data = NULL;
}
}
////////// MemviewSliceCopyTemplate ////////// ////////// MemviewSliceCopyTemplate //////////
static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) { static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
...@@ -259,7 +360,8 @@ static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) { ...@@ -259,7 +360,8 @@ static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
} }
__Pyx_GOTREF(array_obj); __Pyx_GOTREF(array_obj);
memview_obj = __pyx_memoryview_new((PyObject *) array_obj, {{contig_flag}}); memview_obj = (struct __pyx_memoryview_obj *) __pyx_memoryview_new(
(PyObject *) array_obj, {{contig_flag}});
if (unlikely(!memview_obj)) { if (unlikely(!memview_obj)) {
goto fail; goto fail;
} }
...@@ -292,3 +394,21 @@ no_fail: ...@@ -292,3 +394,21 @@ no_fail:
} }
/////////////// MemviewSliceIndex ///////////////
static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset) {
bufp = bufp + idx * stride;
if (suboffset >= 0) {
bufp = *((char **) bufp) + suboffset;
}
return bufp;
}
/* The call has already done the indexing */
static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset) {
if (suboffset >= 0) {
bufp = *((char **) bufp) + suboffset;
}
return bufp;
}
Cython/Utils.py
View file @ 83e7a363
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
# #
import os, sys, re, codecs import os, sys, re, codecs
from Cython import Tempita
def replace_suffix(path, newsuf): def replace_suffix(path, newsuf):
base, _ = os.path.splitext(path) base, _ = os.path.splitext(path)
...@@ -215,10 +216,3 @@ def long_literal(value): ...@@ -215,10 +216,3 @@ def long_literal(value):
if isinstance(value, basestring): if isinstance(value, basestring):
value = str_to_number(value) value = str_to_number(value)
return not -2**31 <= value < 2**31 return not -2**31 <= value < 2**31
def none_or_sub(s, data):
if s is None:
return s
else:
return s % data
tests/run/bufaccess.pyx
View file @ 83e7a363
...@@ -9,13 +9,6 @@ ...@@ -9,13 +9,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from libc cimport stdlib
from libc cimport stdio
cimport cpython.buffer
cimport cython
from cpython cimport PyObject, Py_INCREF, Py_DECREF
__test__ = {} __test__ = {}
import sys import sys
...@@ -33,9 +26,8 @@ def testcase(func): ...@@ -33,9 +26,8 @@ def testcase(func):
__test__[func.__name__] = doctest __test__[func.__name__] = doctest
return func return func
def testcas(a):
pass
include "mockbuffers.pxi"
# #
# Buffer acquire and release tests # Buffer acquire and release tests
...@@ -966,246 +958,6 @@ def buffer_cast_fails(object[char, cast=True] buf): ...@@ -966,246 +958,6 @@ def buffer_cast_fails(object[char, cast=True] buf):
""" """
return buf[0] return buf[0]
#
# Testcase support code (more tests below!, because of scope rules)
#
available_flags = (
('FORMAT', cpython.buffer.PyBUF_FORMAT),
('INDIRECT', cpython.buffer.PyBUF_INDIRECT),
('ND', cpython.buffer.PyBUF_ND),
('STRIDES', cpython.buffer.PyBUF_STRIDES),
('C_CONTIGUOUS', cpython.buffer.PyBUF_C_CONTIGUOUS),
('F_CONTIGUOUS', cpython.buffer.PyBUF_F_CONTIGUOUS),
('WRITABLE', cpython.buffer.PyBUF_WRITABLE)
)
cdef class MockBuffer:
cdef object format, offset
cdef void* buffer
cdef int len, itemsize, ndim
cdef Py_ssize_t* strides
cdef Py_ssize_t* shape
cdef Py_ssize_t* suboffsets
cdef object label, log
cdef readonly object recieved_flags, release_ok
cdef public object fail
def __init__(self, label, data, shape=None, strides=None, format=None, offset=0):
# It is important not to store references to data after the constructor
# as refcounting is checked on object buffers.
self.label = label
self.release_ok = True
self.log = ""
self.offset = offset
self.itemsize = self.get_itemsize()
if format is None: format = self.get_default_format()
if shape is None: shape = (len(data),)
if strides is None:
strides = []
cumprod = 1
rshape = list(shape)
rshape.reverse()
for s in rshape:
strides.append(cumprod)
cumprod *= s
strides.reverse()
strides = [x * self.itemsize for x in strides]
suboffsets = [-1] * len(shape)
datashape = [len(data)]
p = data
while True:
p = p[0]
if isinstance(p, list): datashape.append(len(p))
else: break
if len(datashape) > 1:
# indirect access
self.ndim = len(datashape)
shape = datashape
self.buffer = self.create_indirect_buffer(data, shape)
suboffsets = [0] * (self.ndim-1) + [-1]
strides = [sizeof(void*)] * (self.ndim-1) + [self.itemsize]
self.suboffsets = self.list_to_sizebuf(suboffsets)
else:
# strided and/or simple access
self.buffer = self.create_buffer(data)
self.ndim = len(shape)
self.suboffsets = NULL
try:
format = format.encode('ASCII')
except AttributeError:
pass
self.format = format
self.len = len(data) * self.itemsize
self.strides = self.list_to_sizebuf(strides)
self.shape = self.list_to_sizebuf(shape)
def __dealloc__(self):
stdlib.free(self.strides)
stdlib.free(self.shape)
if self.suboffsets != NULL:
stdlib.free(self.suboffsets)
# must recursively free indirect...
else:
stdlib.free(self.buffer)
cdef void* create_buffer(self, data):
cdef size_t n = <size_t>(len(data) * self.itemsize)
cdef char* buf = <char*>stdlib.malloc(n)
cdef char* it = buf
for value in data:
self.write(it, value)
it += self.itemsize
return buf
cdef void* create_indirect_buffer(self, data, shape):
cdef size_t n = 0
cdef void** buf
assert shape[0] == len(data)
if len(shape) == 1:
return self.create_buffer(data)
else:
shape = shape[1:]
n = <size_t>len(data) * sizeof(void*)
buf = <void**>stdlib.malloc(n)
for idx, subdata in enumerate(data):
buf[idx] = self.create_indirect_buffer(subdata, shape)
return buf
cdef Py_ssize_t* list_to_sizebuf(self, l):
cdef size_t n = <size_t>len(l) * sizeof(Py_ssize_t)
cdef Py_ssize_t* buf = <Py_ssize_t*>stdlib.malloc(n)
for i, x in enumerate(l):
buf[i] = x
return buf
def __getbuffer__(MockBuffer self, Py_buffer* buffer, int flags):
if self.fail:
raise ValueError("Failing on purpose")
self.recieved_flags = []
cdef int value
for name, value in available_flags:
if (value & flags) == value:
self.recieved_flags.append(name)
buffer.buf = <void*>(<char*>self.buffer + (<int>self.offset * self.itemsize))
buffer.obj = self
buffer.len = self.len
buffer.readonly = 0
buffer.format = <char*>self.format
buffer.ndim = self.ndim
buffer.shape = self.shape
buffer.strides = self.strides
buffer.suboffsets = self.suboffsets
buffer.itemsize = self.itemsize
buffer.internal = NULL
if self.label:
msg = "acquired %s" % self.label
print msg
self.log += msg + "\n"
def __releasebuffer__(MockBuffer self, Py_buffer* buffer):
if buffer.suboffsets != self.suboffsets:
self.release_ok = False
if self.label:
msg = "released %s" % self.label
print msg
self.log += msg + "\n"
def printlog(self):
print self.log[:-1]
def resetlog(self):
self.log = ""
cdef int write(self, char* buf, object value) except -1: raise Exception()
cdef get_itemsize(self):
print "ERROR, not subclassed", self.__class__
cdef get_default_format(self):
print "ERROR, not subclassed", self.__class__
cdef class CharMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<char*>buf)[0] = <char>value
return 0
cdef get_itemsize(self): return sizeof(char)
cdef get_default_format(self): return b"@b"
cdef class IntMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<int*>buf)[0] = <int>value
return 0
cdef get_itemsize(self): return sizeof(int)
cdef get_default_format(self): return b"@i"
cdef class UnsignedIntMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<unsigned int*>buf)[0] = <unsigned int>value
return 0
cdef get_itemsize(self): return sizeof(unsigned int)
cdef get_default_format(self): return b"@I"
cdef class ShortMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<short*>buf)[0] = <short>value
return 0
cdef get_itemsize(self): return sizeof(short)
cdef get_default_format(self): return b"h" # Try without endian specifier
cdef class UnsignedShortMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<unsigned short*>buf)[0] = <unsigned short>value
return 0
cdef get_itemsize(self): return sizeof(unsigned short)
cdef get_default_format(self): return b"@1H" # Try with repeat count
cdef class FloatMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<float*>buf)[0] = <float>(<double>value)
return 0
cdef get_itemsize(self): return sizeof(float)
cdef get_default_format(self): return b"f"
cdef class DoubleMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<double*>buf)[0] = <double>value
return 0
cdef get_itemsize(self): return sizeof(double)
cdef get_default_format(self): return b"d"
cdef extern from *:
void* addr_of_pyobject "(void*)"(object)
cdef class ObjectMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<void**>buf)[0] = addr_of_pyobject(value)
return 0
cdef get_itemsize(self): return sizeof(void*)
cdef get_default_format(self): return b"@O"
cdef class IntStridedMockBuffer(IntMockBuffer):
cdef __cythonbufferdefaults__ = {"mode" : "strided"}
cdef class ErrorBuffer:
cdef object label
def __init__(self, label):
self.label = label
def __getbuffer__(ErrorBuffer self, Py_buffer* buffer, int flags):
raise Exception("acquiring %s" % self.label)
def __releasebuffer__(ErrorBuffer self, Py_buffer* buffer):
raise Exception("releasing %s" % self.label)
# #
# Typed buffers # Typed buffers
# #
...@@ -1257,75 +1009,6 @@ def bufdefaults1(IntStridedMockBuffer[int, ndim=1] buf): ...@@ -1257,75 +1009,6 @@ def bufdefaults1(IntStridedMockBuffer[int, ndim=1] buf):
pass pass
#
# Structs
#
cdef struct MyStruct:
char a
char b
long long int c
int d
int e
cdef struct SmallStruct:
int a
int b
cdef struct NestedStruct:
SmallStruct x
SmallStruct y
int z
cdef packed struct PackedStruct:
char a
int b
cdef struct NestedPackedStruct:
char a
int b
PackedStruct sub
int c
cdef class MyStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef MyStruct* s
s = <MyStruct*>buf;
s.a, s.b, s.c, s.d, s.e = value
return 0
cdef get_itemsize(self): return sizeof(MyStruct)
cdef get_default_format(self): return b"2bq2i"
cdef class NestedStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef NestedStruct* s
s = <NestedStruct*>buf;
s.x.a, s.x.b, s.y.a, s.y.b, s.z = value
return 0
cdef get_itemsize(self): return sizeof(NestedStruct)
cdef get_default_format(self): return b"2T{ii}i"
cdef class PackedStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef PackedStruct* s
s = <PackedStruct*>buf;
s.a, s.b = value
return 0
cdef get_itemsize(self): return sizeof(PackedStruct)
cdef get_default_format(self): return b"^ci"
cdef class NestedPackedStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef NestedPackedStruct* s
s = <NestedPackedStruct*>buf;
s.a, s.b, s.sub.a, s.sub.b, s.c = value
return 0
cdef get_itemsize(self): return sizeof(NestedPackedStruct)
cdef get_default_format(self): return b"ci^ci@i"
@testcase @testcase
def basic_struct(object[MyStruct] buf): def basic_struct(object[MyStruct] buf):
""" """
......
tests/run/cythonarray.pyx
View file @ 83e7a363
...@@ -2,10 +2,8 @@ ...@@ -2,10 +2,8 @@
from __future__ import unicode_literals from __future__ import unicode_literals
# from cython cimport array from cython cimport array
# cimport cython.array as array
cimport cython as cy cimport cython as cy
# array = cython.array
def contiguity(): def contiguity():
''' '''
...@@ -69,3 +67,37 @@ def dont_allocate_buffer(): ...@@ -69,3 +67,37 @@ def dont_allocate_buffer():
cdef void callback(char *data): cdef void callback(char *data):
print "callback called %d" % <long> data print "callback called %d" % <long> data
cdef create_array(shape, mode):
cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode)
cdef int *data = <int *> result.data
cdef int i, j, cidx, fidx
for i in range(shape[0]):
for j in range(shape[1]):
cidx = i * shape[1] + j
fidx = i + j * shape[0]
if mode == 'fortran':
data[fidx] = cidx
else:
data[cidx] = cidx
return result
def test_cython_array():
"""
>>> test_cython_array()
98
61
98
61
"""
cdef int[:, ::1] carr = create_array((14, 10), 'c')
cdef int[::1, :] farr = create_array((14, 10), 'fortran')
print carr[9, 8]
print carr[6, 1]
print farr[9, 8]
print farr[6, 1]
tests/run/memslice.pyx 0 → 100644
View file @ 83e7a363
# Tests the buffer access syntax functionality by constructing
# mock buffer objects.
#
# Note that the buffers are mock objects created for testing
# the buffer access behaviour -- for instance there is no flag
# checking in the buffer objects (why test our test case?), rather
# what we want to test is what is passed into the flags argument.
#
from __future__ import unicode_literals
from cython cimport view
__test__ = {}
import sys
import re
exclude = []#re.compile('object').search]
def testcase(func):
for e in exclude:
if e(func.__name__):
return func
doctest = func.__doc__
if sys.version_info >= (3,1,1):
doctest = doctest.replace('does not have the buffer interface',
'does not support the buffer interface')
__test__[func.__name__] = doctest
return func
include "mockbuffers.pxi"
#
# Buffer acquire and release tests
#
def nousage():
"""
The challenge here is just compilation.
"""
cdef int[:, :] buf
@testcase
def acquire_release(o1, o2):
"""
>>> A = IntMockBuffer("A", range(6))
>>> B = IntMockBuffer("B", range(6))
>>> acquire_release(A, B)
acquired A
released A
acquired B
released B
>>> acquire_release(None, None)
>>> acquire_release(None, B)
acquired B
released B
"""
cdef int[:] buf
buf = o1
buf = o2
@testcase
def acquire_raise(o):
"""
Apparently, doctest won't handle mixed exceptions and print
stats, so need to circumvent this.
>>> A = IntMockBuffer("A", range(6))
>>> A.resetlog()
>>> acquire_raise(A)
Traceback (most recent call last):
...
Exception: on purpose
>>> A.printlog()
acquired A
released A
"""
cdef int[:] buf
buf = o
raise Exception("on purpose")
@testcase
def acquire_failure1():
"""
>>> acquire_failure1()
acquired working
0 3
0 3
released working
"""
cdef int[:] buf
buf = IntMockBuffer("working", range(4))
print buf[0], buf[3]
try:
buf = ErrorBuffer()
assert False
except Exception:
print buf[0], buf[3]
@testcase
def acquire_failure2():
"""
>>> acquire_failure2()
acquired working
0 3
0 3
released working
"""
cdef int[:] buf = IntMockBuffer("working", range(4))
print buf[0], buf[3]
try:
buf = ErrorBuffer()
assert False
except Exception:
print buf[0], buf[3]
@testcase
def acquire_failure3():
"""
>>> acquire_failure3()
acquired working
0 3
released working
acquired working
0 3
released working
"""
cdef int[:] buf
buf = IntMockBuffer("working", range(4))
print buf[0], buf[3]
try:
buf = object()
assert False
except Exception:
print buf[0], buf[3]
@testcase
def acquire_nonbuffer1(first, second=None):
"""
>>> acquire_nonbuffer1(3)
Traceback (most recent call last):
...
TypeError: 'int' does not have the buffer interface
>>> acquire_nonbuffer1(type)
Traceback (most recent call last):
...
TypeError: 'type' does not have the buffer interface
>>> acquire_nonbuffer1(None, 2)
Traceback (most recent call last):
...
TypeError: 'int' does not have the buffer interface
"""
cdef int[:] buf
buf = first
buf = second
@testcase
def acquire_nonbuffer2():
"""
>>> acquire_nonbuffer2()
acquired working
0 3
released working
acquired working
0 3
released working
"""
cdef int[:] buf = IntMockBuffer("working", range(4))
print buf[0], buf[3]
try:
buf = ErrorBuffer
assert False
except Exception:
print buf[0], buf[3]
@testcase
def as_argument(int[:] bufarg, int n):
"""
>>> A = IntMockBuffer("A", range(6))
>>> as_argument(A, 6)
acquired A
0 1 2 3 4 5 END
released A
"""
cdef int i
for i in range(n):
print bufarg[i],
print 'END'
@testcase
def as_argument_defval(int[:] bufarg=IntMockBuffer('default', range(6)), int n=6):
"""
>>> as_argument_defval()
acquired default
0 1 2 3 4 5 END
released default
>>> A = IntMockBuffer("A", range(6))
>>> as_argument_defval(A, 6)
acquired A
0 1 2 3 4 5 END
released A
"""
cdef int i
for i in range(n):
print bufarg[i],
print 'END'
@testcase
def cdef_assignment(obj, n):
"""
>>> A = IntMockBuffer("A", range(6))
>>> cdef_assignment(A, 6)
acquired A
0 1 2 3 4 5 END
released A
"""
cdef int[:] buf = obj
cdef int i
for i in range(n):
print buf[i],
print 'END'
@testcase
def forin_assignment(objs, int pick):
"""
>>> A = IntMockBuffer("A", range(6))
>>> B = IntMockBuffer("B", range(6))
>>> forin_assignment([A, B, A, A], 2)
acquired A
2
released A
acquired B
2
released B
acquired A
2
released A
acquired A
2
released A
"""
cdef int[:] buf
for buf in objs:
print buf[pick]
@testcase
def cascaded_buffer_assignment(obj):
"""
>>> A = IntMockBuffer("A", range(6))
>>> cascaded_buffer_assignment(A)
acquired A
acquired A
released A
released A
"""
cdef int[:] a, b
a = b = obj
@testcase
def tuple_buffer_assignment1(a, b):
"""
>>> A = IntMockBuffer("A", range(6))
>>> B = IntMockBuffer("B", range(6))
>>> tuple_buffer_assignment1(A, B)
acquired A
acquired B
released A
released B
"""
cdef int[:] x, y
x, y = a, b
@testcase
def tuple_buffer_assignment2(tup):
"""
>>> A = IntMockBuffer("A", range(6))
>>> B = IntMockBuffer("B", range(6))
>>> tuple_buffer_assignment2((A, B))
acquired A
acquired B
released A
released B
"""
cdef int[:] x, y
x, y = tup
@testcase
def explicitly_release_buffer():
"""
>>> explicitly_release_buffer()
acquired A
released A
After release
"""
cdef int[:] x = IntMockBuffer("A", range(10))
del x
print "After release"
#
# Getting items and index bounds checking
#
@testcase
def get_int_2d(int[:, :] buf, int i, int j):
"""
>>> C = IntMockBuffer("C", range(6), (2,3))
>>> get_int_2d(C, 1, 1)
acquired C
released C
4
Check negative indexing:
>>> get_int_2d(C, -1, 0)
acquired C
released C
3
>>> get_int_2d(C, -1, -2)
acquired C
released C
4
>>> get_int_2d(C, -2, -3)
acquired C
released C
0
Out-of-bounds errors:
>>> get_int_2d(C, 2, 0)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 0)
>>> get_int_2d(C, 0, -4)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 1)
"""
return buf[i, j]
@testcase
def get_int_2d_uintindex(int[:, :] buf, unsigned int i, unsigned int j):
"""
Unsigned indexing:
>>> C = IntMockBuffer("C", range(6), (2,3))
>>> get_int_2d_uintindex(C, 0, 0)
acquired C
released C
0
>>> get_int_2d_uintindex(C, 1, 2)
acquired C
released C
5
"""
# This is most interesting with regards to the C code
# generated.
return buf[i, j]
@testcase
def set_int_2d(int[:, :] buf, int i, int j, int value):
"""
Uses get_int_2d to read back the value afterwards. For pure
unit test, one should support reading in MockBuffer instead.
>>> C = IntMockBuffer("C", range(6), (2,3))
>>> set_int_2d(C, 1, 1, 10)
acquired C
released C
>>> get_int_2d(C, 1, 1)
acquired C
released C
10
Check negative indexing:
>>> set_int_2d(C, -1, 0, 3)
acquired C
released C
>>> get_int_2d(C, -1, 0)
acquired C
released C
3
>>> set_int_2d(C, -1, -2, 8)
acquired C
released C
>>> get_int_2d(C, -1, -2)
acquired C
released C
8
>>> set_int_2d(C, -2, -3, 9)
acquired C
released C
>>> get_int_2d(C, -2, -3)
acquired C
released C
9
Out-of-bounds errors:
>>> set_int_2d(C, 2, 0, 19)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 0)
>>> set_int_2d(C, 0, -4, 19)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 1)
"""
buf[i, j] = value
@testcase
def list_comprehension(int[:] buf, len):
"""
>>> list_comprehension(IntMockBuffer(None, [1,2,3]), 3)
1|2|3
"""
cdef int i
print u"|".join([unicode(buf[i]) for i in range(len)])
#
# The negative_indices buffer option
#
@testcase
def no_negative_indices(object[int, negative_indices=False] buf, int idx):
"""
The most interesting thing here is to inspect the C source and
make sure optimal code is produced.
>>> A = IntMockBuffer(None, range(6))
>>> no_negative_indices(A, 3)
3
>>> no_negative_indices(A, -1)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 0)
"""
return buf[idx]
@testcase
@cython.wraparound(False)
def wraparound_directive(int[:] buf, int pos_idx, int neg_idx):
"""
Again, the most interesting thing here is to inspect the C source.
>>> A = IntMockBuffer(None, range(4))
>>> wraparound_directive(A, 2, -1)
5
>>> wraparound_directive(A, -1, 2)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 0)
"""
cdef int byneg
with cython.wraparound(True):
byneg = buf[neg_idx]
return buf[pos_idx] + byneg
#
# Test which flags are passed.
#
# @testcase
# def readonly(obj):
# """
# >>> R = UnsignedShortMockBuffer("R", range(27), shape=(3, 3, 3))
# >>> readonly(R)
# acquired R
# 25
# released R
# >>> [str(x) for x in R.recieved_flags] # Works in both py2 and py3
# ['FORMAT', 'INDIRECT', 'ND', 'STRIDES']
# """
# cdef unsigned short int[:, :, :] buf = obj
# print buf[2, 2, 1]
@testcase
def writable(obj):
"""
>>> R = UnsignedShortMockBuffer("R", range(27), shape=(3, 3, 3))
>>> writable(R)
acquired R
released R
>>> [str(x) for x in R.recieved_flags] # Py2/3
['FORMAT', 'ND', 'STRIDES', 'WRITABLE']
"""
cdef unsigned short int[:, :, :] buf = obj
buf[2, 2, 1] = 23
@testcase
def strided(int[:] buf):
"""
>>> A = IntMockBuffer("A", range(4))
>>> strided(A)
acquired A
released A
2
>>> [str(x) for x in A.recieved_flags] # Py2/3
['FORMAT', 'ND', 'STRIDES', 'WRITABLE']
Check that the suboffsets were patched back prior to release.
>>> A.release_ok
True
"""
return buf[2]
@testcase
def c_contig(int[::1] buf):
"""
>>> A = IntMockBuffer(None, range(4))
>>> c_contig(A)
2
>>> [str(x) for x in A.recieved_flags]
['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS']
"""
return buf[2]
@testcase
def c_contig_2d(int[:, ::1] buf):
"""
Multi-dim has seperate implementation
>>> A = IntMockBuffer(None, range(12), shape=(3,4))
>>> c_contig_2d(A)
7
>>> [str(x) for x in A.recieved_flags]
['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS']
"""
return buf[1, 3]
@testcase
def f_contig(int[::1, :] buf):
"""
>>> A = IntMockBuffer(None, range(4), shape=(2, 2))
>>> f_contig(A)
2
>>> [str(x) for x in A.recieved_flags]
['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS']
"""
return buf[0, 1]
@testcase
def f_contig_2d(object[int, ndim=2, mode='fortran'] buf):
"""
Must set up strides manually to ensure Fortran ordering.
>>> A = IntMockBuffer(None, range(12), shape=(4,3), strides=(1, 4))
>>> f_contig_2d(A)
7
>>> [str(x) for x in A.recieved_flags]
['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS']
"""
return buf[3, 1]
#
# Test compiler options for bounds checking. We create an array with a
# safe "boundary" (memory
# allocated outside of what it published) and then check whether we get back
# what we stored in the memory or an error.
@testcase
def safe_get(int[:] buf, int idx):
"""
>>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5)
Validate our testing buffer...
>>> safe_get(A, 0)
5
>>> safe_get(A, 2)
7
>>> safe_get(A, -3)
5
Access outside it. This is already done above for bounds check
testing but we include it to tell the story right.
>>> safe_get(A, -4)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 0)
>>> safe_get(A, 3)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 0)
"""
return buf[idx]
@testcase
@cython.boundscheck(False) # outer decorators should take precedence
@cython.boundscheck(True)
def unsafe_get(int[:] buf, int idx):
"""
Access outside of the area the buffer publishes.
>>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5)
>>> unsafe_get(A, -4)
4
>>> unsafe_get(A, -5)
3
>>> unsafe_get(A, 3)
8
"""
return buf[idx]
# @testcase
# @cython.boundscheck(False)
# def unsafe_get_nonegative(object[int, negative_indices=False] buf, int idx):
# """
# Also inspect the C source to see that it is optimal...
#
# >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5)
# >>> unsafe_get_nonegative(A, -2)
# 3
# """
# return buf[idx]
@testcase
def mixed_get(int[:] buf, int unsafe_idx, int safe_idx):
"""
>>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5)
>>> mixed_get(A, -4, 0)
(4, 5)
>>> mixed_get(A, 0, -4)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 0)
"""
with cython.boundscheck(False):
one = buf[unsafe_idx]
with cython.boundscheck(True):
two = buf[safe_idx]
return (one, two)
#
# Coercions
#
## @testcase
## def coercions(object[unsigned char] uc):
## """
## TODO
## """
## print type(uc[0])
## uc[0] = -1
## print uc[0]
## uc[0] = <int>3.14
## print uc[0]
## cdef char* ch = b"asfd"
## cdef object[object] objbuf
## objbuf[3] = ch
#
# Testing that accessing data using various types of buffer access
# all works.
#
def printbuf_int(int[:] buf, shape):
# Utility func
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
@testcase
def printbuf_int_2d(o, shape):
"""
Strided:
>>> printbuf_int_2d(IntMockBuffer("A", range(6), (2,3)), (2,3))
acquired A
0 1 2 END
3 4 5 END
released A
>>> printbuf_int_2d(IntMockBuffer("A", range(100), (3,3), strides=(20,5)), (3,3))
acquired A
0 5 10 END
20 25 30 END
40 45 50 END
released A
Indirect:
>>> printbuf_int_2d(IntMockBuffer("A", [[1,2],[3,4]]), (2,2))
acquired A
1 2 END
3 4 END
released A
"""
# should make shape builtin
cdef int[:, :] buf
buf = o
cdef int i, j
for i in range(shape[0]):
for j in range(shape[1]):
print buf[i, j],
print 'END'
@testcase
def printbuf_float(o, shape):
"""
>>> printbuf_float(FloatMockBuffer("F", [1.0, 1.25, 0.75, 1.0]), (4,))
acquired F
1.0 1.25 0.75 1.0 END
released F
"""
# should make shape builtin
cdef float[:] buf
buf = o
cdef int i, j
for i in range(shape[0]):
print buf[i],
print "END"
#
# Test assignments
#
@testcase
def inplace_operators(int[:] buf):
"""
>>> buf = IntMockBuffer(None, [2, 2])
>>> inplace_operators(buf)
>>> printbuf_int(buf, (2,))
0 3 END
"""
cdef int j = 0
buf[1] += 1
buf[j] *= 2
buf[0] -= 4
#
# Typedefs
#
# Test three layers of typedefs going through a h file for plain int, and
# simply a header file typedef for floats and unsigned.
ctypedef int td_cy_int
cdef extern from "bufaccess.h":
ctypedef td_cy_int td_h_short # Defined as short, but Cython doesn't know this!
ctypedef float td_h_double # Defined as double
ctypedef unsigned int td_h_ushort # Defined as unsigned short
ctypedef td_h_short td_h_cy_short
@testcase
def printbuf_td_cy_int(td_cy_int[:] buf, shape):
"""
>>> printbuf_td_cy_int(IntMockBuffer(None, range(3)), (3,))
0 1 2 END
>>> printbuf_td_cy_int(ShortMockBuffer(None, range(3)), (3,))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'td_cy_int' but got 'short'
"""
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
@testcase
def printbuf_td_h_short(object[td_h_short] buf, shape):
"""
>>> printbuf_td_h_short(ShortMockBuffer(None, range(3)), (3,))
0 1 2 END
>>> printbuf_td_h_short(IntMockBuffer(None, range(3)), (3,))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'td_h_short' but got 'int'
"""
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
@testcase
def printbuf_td_h_cy_short(object[td_h_cy_short] buf, shape):
"""
>>> printbuf_td_h_cy_short(ShortMockBuffer(None, range(3)), (3,))
0 1 2 END
>>> printbuf_td_h_cy_short(IntMockBuffer(None, range(3)), (3,))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'td_h_cy_short' but got 'int'
"""
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
@testcase
def printbuf_td_h_ushort(object[td_h_ushort] buf, shape):
"""
>>> printbuf_td_h_ushort(UnsignedShortMockBuffer(None, range(3)), (3,))
0 1 2 END
>>> printbuf_td_h_ushort(ShortMockBuffer(None, range(3)), (3,))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'td_h_ushort' but got 'short'
"""
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
@testcase
def printbuf_td_h_double(object[td_h_double] buf, shape):
"""
>>> printbuf_td_h_double(DoubleMockBuffer(None, [0.25, 1, 3.125]), (3,))
0.25 1.0 3.125 END
>>> printbuf_td_h_double(FloatMockBuffer(None, [0.25, 1, 3.125]), (3,))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'td_h_double' but got 'float'
"""
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
#
# Object access
#
def addref(*args):
for item in args: Py_INCREF(item)
def decref(*args):
for item in args: Py_DECREF(item)
def get_refcount(x):
return (<PyObject*>x).ob_refcnt
@testcase
def printbuf_object(object[object] buf, shape):
"""
Only play with unique objects, interned numbers etc. will have
unpredictable refcounts.
ObjectMockBuffer doesn't do anything about increfing/decrefing,
we to the "buffer implementor" refcounting directly in the
testcase.
>>> a, b, c = "globally_unique_string_23234123", {4:23}, [34,3]
>>> get_refcount(a), get_refcount(b), get_refcount(c)
(2, 2, 2)
>>> A = ObjectMockBuffer(None, [a, b, c])
>>> printbuf_object(A, (3,))
'globally_unique_string_23234123' 2
{4: 23} 2
[34, 3] 2
"""
cdef int i
for i in range(shape[0]):
print repr(buf[i]), (<PyObject*>buf[i]).ob_refcnt
@testcase
def assign_to_object(object[object] buf, int idx, obj):
"""
See comments on printbuf_object above.
>>> a, b = [1, 2, 3], [4, 5, 6]
>>> get_refcount(a), get_refcount(b)
(2, 2)
>>> addref(a)
>>> A = ObjectMockBuffer(None, [1, a]) # 1, ...,otherwise it thinks nested lists...
>>> get_refcount(a), get_refcount(b)
(3, 2)
>>> assign_to_object(A, 1, b)
>>> get_refcount(a), get_refcount(b)
(2, 3)
>>> decref(b)
"""
buf[idx] = obj
@testcase
def assign_temporary_to_object(object[object] buf):
"""
See comments on printbuf_object above.
>>> a, b = [1, 2, 3], {4:23}
>>> get_refcount(a)
2
>>> addref(a)
>>> A = ObjectMockBuffer(None, [b, a])
>>> get_refcount(a)
3
>>> assign_temporary_to_object(A)
>>> get_refcount(a)
2
>>> printbuf_object(A, (2,))
{4: 23} 2
{1: 8} 2
To avoid leaking a reference in our testcase we need to
replace the temporary with something we can manually decref :-)
>>> assign_to_object(A, 1, a)
>>> decref(a)
"""
buf[1] = {3-2: 2+(2*4)-2}
#
# cast option
#
@testcase
def buffer_cast(object[unsigned int, cast=True] buf, int idx):
"""
Round-trip a signed int through unsigned int buffer access.
>>> A = IntMockBuffer(None, [-100])
>>> buffer_cast(A, 0)
-100
"""
cdef unsigned int data = buf[idx]
return <int>data
@testcase
def buffer_cast_fails(object[char, cast=True] buf):
"""
Cannot cast between datatype of different sizes.
>>> buffer_cast_fails(IntMockBuffer(None, [0]))
Traceback (most recent call last):
...
ValueError: Item size of buffer (4 bytes) does not match size of 'char' (1 byte)
"""
return buf[0]
#
# Typed buffers
#
@testcase
def typedbuffer1(obj):
"""
>>> typedbuffer1(IntMockBuffer("A", range(10)))
acquired A
released A
>>> typedbuffer1(None)
>>> typedbuffer1(4)
Traceback (most recent call last):
...
TypeError: Cannot convert int to memslice.IntMockBuffer
"""
cdef IntMockBuffer[int, ndim=1] buf = obj
@testcase
def typedbuffer2(IntMockBuffer[int, ndim=1] obj):
"""
>>> typedbuffer2(IntMockBuffer("A", range(10)))
acquired A
released A
>>> typedbuffer2(None)
>>> typedbuffer2(4)
Traceback (most recent call last):
...
TypeError: Argument 'obj' has incorrect type (expected memslice.IntMockBuffer, got int)
"""
pass
#
# Test __cythonbufferdefaults__
#
@testcase
def bufdefaults1(IntStridedMockBuffer[int, ndim=1] buf):
"""
For IntStridedMockBuffer, mode should be
"strided" by defaults which should show
up in the flags.
>>> A = IntStridedMockBuffer("A", range(10))
>>> bufdefaults1(A)
acquired A
released A
>>> [str(x) for x in A.recieved_flags]
['FORMAT', 'ND', 'STRIDES']
"""
pass
@testcase
def basic_struct(object[MyStruct] buf):
"""
See also buffmt.pyx
>>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
1 2 3 4 5
>>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="bbqii"))
1 2 3 4 5
"""
print buf[0].a, buf[0].b, buf[0].c, buf[0].d, buf[0].e
@testcase
def nested_struct(object[NestedStruct] buf):
"""
See also buffmt.pyx
>>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
1 2 3 4 5
>>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i"))
1 2 3 4 5
"""
print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z
@testcase
def packed_struct(object[PackedStruct] buf):
"""
See also buffmt.pyx
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)]))
1 2
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c^i}"))
1 2
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c=i}"))
1 2
"""
print buf[0].a, buf[0].b
@testcase
def nested_packed_struct(object[NestedPackedStruct] buf):
"""
See also buffmt.pyx
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
1 2 3 4 5
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ci^ci@i"))
1 2 3 4 5
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i"))
1 2 3 4 5
"""
print buf[0].a, buf[0].b, buf[0].sub.a, buf[0].sub.b, buf[0].c
cdef struct LongComplex:
long double real
long double imag
cdef class LongComplexMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef LongComplex* s
s = <LongComplex*>buf;
s.real, s.imag = value
return 0
cdef get_itemsize(self): return sizeof(LongComplex)
cdef get_default_format(self): return b"Zg"
#cdef extern from "complex.h":
# pass
@testcase
def complex_dtype(object[long double complex] buf):
"""
>>> complex_dtype(LongComplexMockBuffer(None, [(0, -1)]))
-1j
"""
print buf[0]
@testcase
def complex_inplace(object[long double complex] buf):
"""
>>> complex_inplace(LongComplexMockBuffer(None, [(0, -1)]))
(1+1j)
"""
buf[0] = buf[0] + 1 + 2j
print buf[0]
@testcase
def complex_struct_dtype(object[LongComplex] buf):
"""
Note that the format string is "Zg" rather than "2g", yet a struct
is accessed.
>>> complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)]))
0.0 -1.0
"""
print buf[0].real, buf[0].imag
@testcase
def complex_struct_inplace(object[LongComplex] buf):
"""
>>> complex_struct_inplace(LongComplexMockBuffer(None, [(0, -1)]))
1.0 1.0
"""
buf[0].real += 1
buf[0].imag += 2
print buf[0].real, buf[0].imag
#
# Nogil
#
@testcase
@cython.boundscheck(False)
def buffer_nogil():
"""
>>> buffer_nogil()
10
"""
cdef int[:] buf = IntMockBuffer(None, [1,2,3])
with nogil:
buf[1] = 10
return buf[1]
tests/run/memslice_indexing.pyx deleted 100644 → 0
View file @ aad49345
cimport cython
from cython cimport array
from libc.stdlib cimport malloc, free
def create_array(shape, mode='c'):
cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode)
cdef int *data = <int *> result.data
cdef int i, j, value
for i in range(shape[0]):
for j in range(shape[1]):
value = i * shape[0] + j
if mode == 'fortran':
data[i + j * 10] = value
else:
data[value] = value
return result
def slice_contig_indexing():
"""
>>> print("disabled")
disabled
slice_contig_indexing()
98
61
98
61
"""
cdef int[:, ::1] carr = create_array((14, 10))
cdef int[::1, :] farr = create_array((10, 14), mode='fortran')
print carr[9, 8]
print carr[6, 1]
print farr[9, 8]
print farr[6, 1]
tests/run/mockbuffers.pxi 0 → 100644
View file @ 83e7a363
from libc cimport stdlib
from libc cimport stdio
cimport cpython.buffer
cimport cython
from cpython cimport PyObject, Py_INCREF, Py_DECREF
available_flags = (
('FORMAT', cpython.buffer.PyBUF_FORMAT),
('INDIRECT', cpython.buffer.PyBUF_INDIRECT),
('ND', cpython.buffer.PyBUF_ND),
('STRIDES', cpython.buffer.PyBUF_STRIDES),
('C_CONTIGUOUS', cpython.buffer.PyBUF_C_CONTIGUOUS),
('F_CONTIGUOUS', cpython.buffer.PyBUF_F_CONTIGUOUS),
('WRITABLE', cpython.buffer.PyBUF_WRITABLE)
)
cdef class MockBuffer:
cdef object format, offset
cdef void* buffer
cdef int len, itemsize, ndim
cdef Py_ssize_t* strides
cdef Py_ssize_t* shape
cdef Py_ssize_t* suboffsets
cdef object label, log
cdef readonly object recieved_flags, release_ok
cdef public object fail
def __init__(self, label, data, shape=None, strides=None, format=None, offset=0):
# It is important not to store references to data after the constructor
# as refcounting is checked on object buffers.
self.label = label
self.release_ok = True
self.log = ""
self.offset = offset
self.itemsize = self.get_itemsize()
if format is None: format = self.get_default_format()
if shape is None: shape = (len(data),)
if strides is None:
strides = []
cumprod = 1
rshape = list(shape)
rshape.reverse()
for s in rshape:
strides.append(cumprod)
cumprod *= s
strides.reverse()
strides = [x * self.itemsize for x in strides]
suboffsets = [-1] * len(shape)
datashape = [len(data)]
p = data
while True:
p = p[0]
if isinstance(p, list): datashape.append(len(p))
else: break
if len(datashape) > 1:
# indirect access
self.ndim = len(datashape)
shape = datashape
self.buffer = self.create_indirect_buffer(data, shape)
suboffsets = [0] * (self.ndim-1) + [-1]
strides = [sizeof(void*)] * (self.ndim-1) + [self.itemsize]
self.suboffsets = self.list_to_sizebuf(suboffsets)
else:
# strided and/or simple access
self.buffer = self.create_buffer(data)
self.ndim = len(shape)
self.suboffsets = NULL
try:
format = format.encode('ASCII')
except AttributeError:
pass
self.format = format
self.len = len(data) * self.itemsize
self.strides = self.list_to_sizebuf(strides)
self.shape = self.list_to_sizebuf(shape)
def __dealloc__(self):
stdlib.free(self.strides)
stdlib.free(self.shape)
if self.suboffsets != NULL:
stdlib.free(self.suboffsets)
# must recursively free indirect...
else:
stdlib.free(self.buffer)
cdef void* create_buffer(self, data) except NULL:
cdef size_t n = <size_t>(len(data) * self.itemsize)
cdef char* buf = <char*>stdlib.malloc(n)
if buf == NULL:
raise MemoryError
cdef char* it = buf
for value in data:
self.write(it, value)
it += self.itemsize
return buf
cdef void* create_indirect_buffer(self, data, shape):
cdef size_t n = 0
cdef void** buf
assert shape[0] == len(data)
if len(shape) == 1:
return self.create_buffer(data)
else:
shape = shape[1:]
n = <size_t>len(data) * sizeof(void*)
buf = <void**>stdlib.malloc(n)
for idx, subdata in enumerate(data):
buf[idx] = self.create_indirect_buffer(subdata, shape)
return buf
cdef Py_ssize_t* list_to_sizebuf(self, l):
cdef size_t n = <size_t>len(l) * sizeof(Py_ssize_t)
cdef Py_ssize_t* buf = <Py_ssize_t*>stdlib.malloc(n)
for i, x in enumerate(l):
buf[i] = x
return buf
def __getbuffer__(MockBuffer self, Py_buffer* buffer, int flags):
if self.fail:
raise ValueError("Failing on purpose")
self.recieved_flags = []
cdef int value
for name, value in available_flags:
if (value & flags) == value:
self.recieved_flags.append(name)
buffer.buf = <void*>(<char*>self.buffer + (<int>self.offset * self.itemsize))
buffer.obj = self
buffer.len = self.len
buffer.readonly = 0
buffer.format = <char*>self.format
buffer.ndim = self.ndim
buffer.shape = self.shape
buffer.strides = self.strides
buffer.suboffsets = self.suboffsets
buffer.itemsize = self.itemsize
buffer.internal = NULL
if self.label:
msg = "acquired %s" % self.label
print msg
self.log += msg + "\n"
def __releasebuffer__(MockBuffer self, Py_buffer* buffer):
if buffer.suboffsets != self.suboffsets:
self.release_ok = False
if self.label:
msg = "released %s" % self.label
print msg
self.log += msg + "\n"
def printlog(self):
print self.log[:-1]
def resetlog(self):
self.log = ""
cdef int write(self, char* buf, object value) except -1: raise Exception()
cdef get_itemsize(self):
print "ERROR, not subclassed", self.__class__
cdef get_default_format(self):
print "ERROR, not subclassed", self.__class__
cdef class CharMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<char*>buf)[0] = <char>value
return 0
cdef get_itemsize(self): return sizeof(char)
cdef get_default_format(self): return b"@b"
cdef class IntMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<int*>buf)[0] = <int>value
return 0
cdef get_itemsize(self): return sizeof(int)
cdef get_default_format(self): return b"@i"
cdef class UnsignedIntMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<unsigned int*>buf)[0] = <unsigned int>value
return 0
cdef get_itemsize(self): return sizeof(unsigned int)
cdef get_default_format(self): return b"@I"
cdef class ShortMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<short*>buf)[0] = <short>value
return 0
cdef get_itemsize(self): return sizeof(short)
cdef get_default_format(self): return b"h" # Try without endian specifier
cdef class UnsignedShortMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<unsigned short*>buf)[0] = <unsigned short>value
return 0
cdef get_itemsize(self): return sizeof(unsigned short)
cdef get_default_format(self): return b"@1H" # Try with repeat count
cdef class FloatMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<float*>buf)[0] = <float>(<double>value)
return 0
cdef get_itemsize(self): return sizeof(float)
cdef get_default_format(self): return b"f"
cdef class DoubleMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<double*>buf)[0] = <double>value
return 0
cdef get_itemsize(self): return sizeof(double)
cdef get_default_format(self): return b"d"
cdef extern from *:
void* addr_of_pyobject "(void*)"(object)
cdef class ObjectMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
(<void**>buf)[0] = addr_of_pyobject(value)
return 0
cdef get_itemsize(self): return sizeof(void*)
cdef get_default_format(self): return b"@O"
cdef class IntStridedMockBuffer(IntMockBuffer):
cdef __cythonbufferdefaults__ = {"mode" : "strided"}
cdef class ErrorBuffer:
cdef object label
def __init__(self, label):
self.label = label
def __getbuffer__(ErrorBuffer self, Py_buffer* buffer, int flags):
raise Exception("acquiring %s" % self.label)
def __releasebuffer__(ErrorBuffer self, Py_buffer* buffer):
raise Exception("releasing %s" % self.label)
#
# Structs
#
cdef struct MyStruct:
char a
char b
long long int c
int d
int e
cdef struct SmallStruct:
int a
int b
cdef struct NestedStruct:
SmallStruct x
SmallStruct y
int z
cdef packed struct PackedStruct:
char a
int b
cdef struct NestedPackedStruct:
char a
int b
PackedStruct sub
int c
cdef class MyStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef MyStruct* s
s = <MyStruct*>buf;
s.a, s.b, s.c, s.d, s.e = value
return 0
cdef get_itemsize(self): return sizeof(MyStruct)
cdef get_default_format(self): return b"2bq2i"
cdef class NestedStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef NestedStruct* s
s = <NestedStruct*>buf;
s.x.a, s.x.b, s.y.a, s.y.b, s.z = value
return 0
cdef get_itemsize(self): return sizeof(NestedStruct)
cdef get_default_format(self): return b"2T{ii}i"
cdef class PackedStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef PackedStruct* s
s = <PackedStruct*>buf;
s.a, s.b = value
return 0
cdef get_itemsize(self): return sizeof(PackedStruct)
cdef get_default_format(self): return b"^ci"
cdef class NestedPackedStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef NestedPackedStruct* s
s = <NestedPackedStruct*>buf;
s.a, s.b, s.sub.a, s.sub.b, s.c = value
return 0
cdef get_itemsize(self): return sizeof(NestedPackedStruct)
cdef get_default_format(self): return b"ci^ci@i"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7