Commit 83e7a363 authored by Mark Florisson's avatar Mark Florisson

MemoryViewSlice indexing and object coercion + MemoryView indexing

parent aad49345
...@@ -215,6 +215,50 @@ class BufferEntry(object): ...@@ -215,6 +215,50 @@ class BufferEntry(object):
def _for_all_ndim(self, s): def _for_all_ndim(self, s):
return [s % (self.cname, i) for i in range(self.type.ndim)] return [s % (self.cname, i) for i in range(self.type.ndim)]
def generate_buffer_lookup_code(self, code, index_cnames):
# Create buffer lookup and return it
# This is done via utility macros/inline functions, which vary
# according to the access mode used.
params = []
nd = self.type.ndim
mode = self.type.mode
if mode == 'full':
for i, s, o in zip(index_cnames,
self.get_buf_stridevars(),
self.get_buf_suboffsetvars()):
params.append(i)
params.append(s)
params.append(o)
funcname = "__Pyx_BufPtrFull%dd" % nd
funcgen = buf_lookup_full_code
else:
if mode == 'strided':
funcname = "__Pyx_BufPtrStrided%dd" % nd
funcgen = buf_lookup_strided_code
elif mode == 'c':
funcname = "__Pyx_BufPtrCContig%dd" % nd
funcgen = buf_lookup_c_code
elif mode == 'fortran':
funcname = "__Pyx_BufPtrFortranContig%dd" % nd
funcgen = buf_lookup_fortran_code
else:
assert False
for i, s in zip(index_cnames, self.get_buf_stridevars()):
params.append(i)
params.append(s)
# Make sure the utility code is available
if funcname not in code.globalstate.utility_codes:
code.globalstate.utility_codes.add(funcname)
protocode = code.globalstate['utility_code_proto']
defcode = code.globalstate['utility_code_def']
funcgen(protocode, defcode, name=funcname, nd=nd)
buf_ptr_type_code = self.buf_ptr_type.declaration_code("")
ptrcode = "%s(%s, %s, %s)" % (funcname, buf_ptr_type_code, self.buf_ptr,
", ".join(params))
return ptrcode
def get_flags(buffer_aux, buffer_type): def get_flags(buffer_aux, buffer_type):
flags = 'PyBUF_FORMAT' flags = 'PyBUF_FORMAT'
...@@ -426,48 +470,7 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives, ...@@ -426,48 +470,7 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives,
if signed != 0: if signed != 0:
code.putln("if (%s < 0) %s += %s;" % (cname, cname, shape)) code.putln("if (%s < 0) %s += %s;" % (cname, cname, shape))
# Create buffer lookup and return it return entry.generate_buffer_lookup_code(code, index_cnames)
# This is done via utility macros/inline functions, which vary
# according to the access mode used.
params = []
nd = entry.type.ndim
mode = entry.type.mode
if mode == 'full':
for i, s, o in zip(index_cnames,
entry.get_buf_stridevars(),
entry.get_buf_suboffsetvars()):
params.append(i)
params.append(s)
params.append(o)
funcname = "__Pyx_BufPtrFull%dd" % nd
funcgen = buf_lookup_full_code
else:
if mode == 'strided':
funcname = "__Pyx_BufPtrStrided%dd" % nd
funcgen = buf_lookup_strided_code
elif mode == 'c':
funcname = "__Pyx_BufPtrCContig%dd" % nd
funcgen = buf_lookup_c_code
elif mode == 'fortran':
funcname = "__Pyx_BufPtrFortranContig%dd" % nd
funcgen = buf_lookup_fortran_code
else:
assert False
for i, s in zip(index_cnames, entry.get_buf_stridevars()):
params.append(i)
params.append(s)
# Make sure the utility code is available
if funcname not in code.globalstate.utility_codes:
code.globalstate.utility_codes.add(funcname)
protocode = code.globalstate['utility_code_proto']
defcode = code.globalstate['utility_code_def']
funcgen(protocode, defcode, name=funcname, nd=nd)
buf_ptr_type_code = entry.buf_ptr_type.declaration_code("")
ptrcode = "%s(%s, %s, %s)" % (funcname, buf_ptr_type_code, entry.buf_ptr,
", ".join(params))
return ptrcode
def use_bufstruct_declare_code(env): def use_bufstruct_declare_code(env):
......
...@@ -23,7 +23,6 @@ import DebugFlags ...@@ -23,7 +23,6 @@ import DebugFlags
import Errors import Errors
from Cython import Tempita as tempita from Cython import Tempita as tempita
from Cython.Utils import none_or_sub
try: try:
from __builtin__ import basestring from __builtin__ import basestring
except ImportError: except ImportError:
...@@ -162,11 +161,11 @@ class UtilityCodeBase(object): ...@@ -162,11 +161,11 @@ class UtilityCodeBase(object):
kwargs['impl'] = impl kwargs['impl'] = impl
if 'name' not in kwargs: if 'name' not in kwargs:
if from_file:
kwargs['name'] = os.path.splitext(from_file)[0]
else:
kwargs['name'] = util_code_name kwargs['name'] = util_code_name
if 'file' not in kwargs and from_file:
kwargs['file'] = from_file
return cls(**kwargs) return cls(**kwargs)
load = classmethod(load) load = classmethod(load)
...@@ -188,13 +187,8 @@ class UtilityCodeBase(object): ...@@ -188,13 +187,8 @@ class UtilityCodeBase(object):
proto, impl = utilities[util_code_name] proto, impl = utilities[util_code_name]
if context is not None: if context is not None:
if '__name' not in context: proto = sub_tempita(proto, context, from_file, util_code_name)
context['__name'] = util_code_name impl = sub_tempita(impl, context, from_file, util_code_name)
if proto:
proto = tempita.sub(proto, **context)
if impl:
impl = tempita.sub(impl, **context)
if cls.is_cython_utility: if cls.is_cython_utility:
# Remember line numbers # Remember line numbers
...@@ -204,19 +198,56 @@ class UtilityCodeBase(object): ...@@ -204,19 +198,56 @@ class UtilityCodeBase(object):
load_as_string = classmethod(load_as_string) load_as_string = classmethod(load_as_string)
def none_or_sub(self, s, context, tempita):
"""
Format a string in this utility code with context. If None, do nothing.
"""
if s is None:
return None
if tempita:
return sub_tempita(s, context, self.file, self.name)
return s % context
def __str__(self): def __str__(self):
return "<%s(%s)" % (type(self).__name__, self.name) return "<%s(%s)" % (type(self).__name__, self.name)
def sub_tempita(s, context, file, name):
"Run tempita on string s with context context."
if not s:
return None
if file:
context['__name'] = "%s:%s" % (file, name)
elif name:
context['__name'] = name
return tempita.sub(s, **context)
class UtilityCode(UtilityCodeBase): class UtilityCode(UtilityCodeBase):
# Stores utility code to add during code generation. """
# Stores utility code to add during code generation.
# See GlobalState.put_utility_code.
# See GlobalState.put_utility_code.
# hashes/equals by instance
hashes/equals by instance
proto C prototypes
impl implemenation code
init code to call on module initialization
requires utility code dependencies
proto_block the place in the resulting file where the prototype should
end up
name name of the utility code (or None)
file filename of the utility code file this utility was loaded
from (or None)
"""
def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None, def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
proto_block='utility_code_proto', name=None): proto_block='utility_code_proto', name=None, file=None):
# proto_block: Which code block to dump prototype in. See GlobalState. # proto_block: Which code block to dump prototype in. See GlobalState.
self.proto = proto self.proto = proto
self.impl = impl self.impl = impl
...@@ -227,11 +258,13 @@ class UtilityCode(UtilityCodeBase): ...@@ -227,11 +258,13 @@ class UtilityCode(UtilityCodeBase):
self.specialize_list = [] self.specialize_list = []
self.proto_block = proto_block self.proto_block = proto_block
self.name = name self.name = name
self.file = file
def get_tree(self): def get_tree(self):
pass pass
def specialize(self, pyrex_type=None, **data):
def specialize(self, pyrex_type=None, tempita=False, **data):
# Dicts aren't hashable... # Dicts aren't hashable...
if pyrex_type is not None: if pyrex_type is not None:
data['type'] = pyrex_type.declaration_code('') data['type'] = pyrex_type.declaration_code('')
...@@ -244,12 +277,15 @@ class UtilityCode(UtilityCodeBase): ...@@ -244,12 +277,15 @@ class UtilityCode(UtilityCodeBase):
requires = None requires = None
else: else:
requires = [r.specialize(data) for r in self.requires] requires = [r.specialize(data) for r in self.requires]
s = self._cache[key] = UtilityCode( s = self._cache[key] = UtilityCode(
none_or_sub(self.proto, data), self.none_or_sub(self.proto, data, tempita),
none_or_sub(self.impl, data), self.none_or_sub(self.impl, data, tempita),
none_or_sub(self.init, data), self.none_or_sub(self.init, data, tempita),
none_or_sub(self.cleanup, data), self.none_or_sub(self.cleanup, data, tempita),
requires, self.proto_block) requires,
self.proto_block)
self.specialize_list.append(s) self.specialize_list.append(s)
return s return s
...@@ -275,6 +311,29 @@ class UtilityCode(UtilityCodeBase): ...@@ -275,6 +311,29 @@ class UtilityCode(UtilityCodeBase):
self.cleanup(writer, output.module_pos) self.cleanup(writer, output.module_pos)
class ContentHashingUtilityCode(UtilityCode):
"UtilityCode that hashes and compares based on self.proto and self.impl"
def __hash__(self):
return hash((self.proto, self.impl))
def __eq__(self, other):
return (self.proto, self.impl) == (other.proto, other.impl)
class LazyUtilityCode(UtilityCodeBase):
"""
Utility code that calls a callback with the root code writer when
available. Useful when you only have 'env' but not 'code'.
"""
def __init__(self, callback):
self.callback = callback
def put_code(self, globalstate):
utility = self.callback(globalstate.rootwriter)
globalstate.use_utility_code(utility)
class FunctionState(object): class FunctionState(object):
# return_label string function return point label # return_label string function return point label
...@@ -1538,6 +1597,15 @@ class CCodeWriter(object): ...@@ -1538,6 +1597,15 @@ class CCodeWriter(object):
for entry in entries: for entry in entries:
self.put_var_xdecref_clear(entry) self.put_var_xdecref_clear(entry)
def put_incref_memoryviewslice(self, slice_cname, have_gil=False):
self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))
def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False):
self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))
def put_xgiveref_memoryviewslice(self, slice_cname):
self.put_xgiveref("%s.memview" % slice_cname)
def put_init_to_py_none(self, cname, type, nanny=True): def put_init_to_py_none(self, cname, type, nanny=True):
from PyrexTypes import py_object_type, typecast from PyrexTypes import py_object_type, typecast
py_none = typecast(type, py_object_type, "Py_None") py_none = typecast(type, py_object_type, "Py_None")
......
...@@ -10,10 +10,12 @@ import MemoryView ...@@ -10,10 +10,12 @@ import MemoryView
class CythonScope(ModuleScope): class CythonScope(ModuleScope):
is_cython_builtin = 1 is_cython_builtin = 1
def __init__(self): def __init__(self, context):
ModuleScope.__init__(self, u'cython', None, None) ModuleScope.__init__(self, u'cython', None, None)
self.pxd_file_loaded = True self.pxd_file_loaded = True
self.populate_cython_scope() self.populate_cython_scope()
# The Main.Context object
self.context = context
def lookup_type(self, name): def lookup_type(self, name):
# This function should go away when types are all first-level objects. # This function should go away when types are all first-level objects.
...@@ -80,6 +82,10 @@ class CythonScope(ModuleScope): ...@@ -80,6 +82,10 @@ class CythonScope(ModuleScope):
# The view sub-scope # The view sub-scope
# #
self.viewscope = viewscope = ModuleScope(u'cython.view', self, None) self.viewscope = viewscope = ModuleScope(u'cython.view', self, None)
# Hacky monkey patch
self.viewscope.global_scope = self.global_scope
self.declare_module('view', viewscope, None) self.declare_module('view', viewscope, None)
viewscope.is_cython_builtin = True viewscope.is_cython_builtin = True
viewscope.pxd_file_loaded = True viewscope.pxd_file_loaded = True
...@@ -93,7 +99,7 @@ def create_cython_scope(context, create_testscope): ...@@ -93,7 +99,7 @@ def create_cython_scope(context, create_testscope):
# One could in fact probably make it a singleton, # One could in fact probably make it a singleton,
# but not sure yet whether any code mutates it (which would kill reusing # but not sure yet whether any code mutates it (which would kill reusing
# it across different contexts) # it across different contexts)
scope = CythonScope() scope = CythonScope(context)
if create_testscope: if create_testscope:
scope.test_cythonscope() scope.test_cythonscope()
...@@ -129,6 +135,12 @@ cython_test_extclass_utility_code = \ ...@@ -129,6 +135,12 @@ cython_test_extclass_utility_code = \
cythonview_testscope_utility_code = load_testscope_utility("View.TestScope") cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
view_utility_code = MemoryView.load_memview_cy_utility( view_utility_code = MemoryView.load_memview_cy_utility(
"View.MemoryView", requires=(Buffer.GetAndReleaseBufferUtilityCode(),)) "View.MemoryView", context=MemoryView.context,
requires=[Buffer.GetAndReleaseBufferUtilityCode(),
cython_array_utility_code = MemoryView.load_memview_cy_utility("CythonArray") MemoryView.memviewslice_declare_code],
)
cython_array_utility_code = MemoryView.load_memview_cy_utility(
"CythonArray",
context=MemoryView.context,
requires=[view_utility_code])
...@@ -587,7 +587,12 @@ class ExprNode(Node): ...@@ -587,7 +587,12 @@ class ExprNode(Node):
if dst_type.is_memoryviewslice: if dst_type.is_memoryviewslice:
import MemoryView import MemoryView
if not src.type.is_memoryviewslice: if not src.type.is_memoryviewslice:
if src.type.is_pyobject:
src = CoerceToMemViewSliceNode(src, dst_type, env) src = CoerceToMemViewSliceNode(src, dst_type, env)
else:
error(self.pos,
"Cannot convert '%s' to memoryviewslice" %
(src_type,))
elif not MemoryView.src_conforms_to_dst(src.type, dst_type): elif not MemoryView.src_conforms_to_dst(src.type, dst_type):
error(self.pos, "Memoryview '%s' not conformable to memoryview '%s'." % error(self.pos, "Memoryview '%s' not conformable to memoryview '%s'." %
(src.type, dst_type)) (src.type, dst_type))
...@@ -1282,6 +1287,7 @@ class NameNode(AtomicExprNode): ...@@ -1282,6 +1287,7 @@ class NameNode(AtomicExprNode):
# cf_is_null boolean Is uninitialized before this node # cf_is_null boolean Is uninitialized before this node
# cf_maybe_null boolean Maybe uninitialized before this node # cf_maybe_null boolean Maybe uninitialized before this node
# allow_null boolean Don't raise UnboundLocalError # allow_null boolean Don't raise UnboundLocalError
# nogil boolean Whether it is used in a nogil context
is_name = True is_name = True
is_cython_module = False is_cython_module = False
...@@ -1293,6 +1299,7 @@ class NameNode(AtomicExprNode): ...@@ -1293,6 +1299,7 @@ class NameNode(AtomicExprNode):
cf_maybe_null = True cf_maybe_null = True
cf_is_null = False cf_is_null = False
allow_null = False allow_null = False
nogil = False
def create_analysed_rvalue(pos, env, entry): def create_analysed_rvalue(pos, env, entry):
node = NameNode(pos) node = NameNode(pos)
...@@ -1452,6 +1459,7 @@ class NameNode(AtomicExprNode): ...@@ -1452,6 +1459,7 @@ class NameNode(AtomicExprNode):
self.is_used_as_rvalue = 1 self.is_used_as_rvalue = 1
def nogil_check(self, env): def nogil_check(self, env):
self.nogil = True
if self.is_used_as_rvalue: if self.is_used_as_rvalue:
entry = self.entry entry = self.entry
if entry.is_builtin: if entry.is_builtin:
...@@ -1655,9 +1663,16 @@ class NameNode(AtomicExprNode): ...@@ -1655,9 +1663,16 @@ class NameNode(AtomicExprNode):
else: else:
if self.type.is_memoryviewslice: if self.type.is_memoryviewslice:
import MemoryView import MemoryView
MemoryView.gen_acquire_memoryviewslice(rhs, self.type, MemoryView.put_acquire_memoryviewslice(rhs, self.type,
self.entry.is_cglobal, self.result(), self.pos, code) self.entry.is_cglobal, self.result(), self.pos, code)
# self.generate_acquire_memoryviewslice(rhs, code)
if isinstance(rhs, CoerceToMemViewSliceNode):
# We had a new reference, the lhs now has another,
# dispose of ours.
# code.put_xdecref_memoryviewslice(rhs.result())
code.put_decref("%s.memview" % rhs.result(),
cython_memoryview_ptr_type,
nanny=False)
elif self.type.is_buffer: elif self.type.is_buffer:
# Generate code for doing the buffer release/acquisition. # Generate code for doing the buffer release/acquisition.
...@@ -1736,12 +1751,17 @@ class NameNode(AtomicExprNode): ...@@ -1736,12 +1751,17 @@ class NameNode(AtomicExprNode):
'__Pyx_DelAttrString(%s, "%s")' % ( '__Pyx_DelAttrString(%s, "%s")' % (
Naming.module_cname, Naming.module_cname,
self.entry.name)) self.entry.name))
elif self.entry.type.is_pyobject: elif self.entry.type.is_pyobject or self.entry.type.is_memoryviewslice:
if not self.cf_is_null: if not self.cf_is_null:
if self.cf_maybe_null: if self.cf_maybe_null:
code.put_error_if_unbound(self.pos, self.entry) code.put_error_if_unbound(self.pos, self.entry)
if self.entry.type.is_pyobject:
code.put_decref(self.result(), self.ctype()) code.put_decref(self.result(), self.ctype())
code.putln('%s = NULL;' % self.result()) code.putln('%s = NULL;' % self.result())
else:
code.put_xdecref_memoryviewslice(self.entry.cname,
have_gil=not self.nogil)
else: else:
error(self.pos, "Deletion of C names not supported") error(self.pos, "Deletion of C names not supported")
...@@ -2347,10 +2367,12 @@ class IndexNode(ExprNode): ...@@ -2347,10 +2367,12 @@ class IndexNode(ExprNode):
buffer_access = False buffer_access = False
memoryviewslice_access = False memoryviewslice_access = False
if (self.base.type.is_memoryviewslice and not self.indices and is_memslice = self.base.type.is_memoryviewslice
if (is_memslice and not self.indices and
isinstance(self.index, EllipsisNode)): isinstance(self.index, EllipsisNode)):
memoryviewslice_access = True memoryviewslice_access = True
elif self.base.type.is_buffer or self.base.type.is_memoryviewslice: elif self.base.type.is_buffer or is_memslice:
if self.indices: if self.indices:
indices = self.indices indices = self.indices
else: else:
...@@ -2358,6 +2380,7 @@ class IndexNode(ExprNode): ...@@ -2358,6 +2380,7 @@ class IndexNode(ExprNode):
indices = self.index.args indices = self.index.args
else: else:
indices = [self.index] indices = [self.index]
if len(indices) == self.base.type.ndim: if len(indices) == self.base.type.ndim:
buffer_access = True buffer_access = True
skip_child_analysis = True skip_child_analysis = True
...@@ -2365,15 +2388,10 @@ class IndexNode(ExprNode): ...@@ -2365,15 +2388,10 @@ class IndexNode(ExprNode):
x.analyse_types(env) x.analyse_types(env)
if not x.type.is_int: if not x.type.is_int:
buffer_access = False buffer_access = False
if buffer_access: if buffer_access:
assert hasattr(self.base, "entry") # Must be a NameNode-like node assert hasattr(self.base, "entry") # Must be a NameNode-like node
# if self.base.type.is_memoryviewslice:
# assert hasattr(self.base, "entry")
# if self.indices or not isinstance(self.index, EllipsisNode):
# error(self.pos, "Memoryviews currently support ellipsis indexing only.")
# else: memoryviewslice_access = True
# On cloning, indices is cloned. Otherwise, unpack index into indices # On cloning, indices is cloned. Otherwise, unpack index into indices
assert not (buffer_access and isinstance(self.index, CloneNode)) assert not (buffer_access and isinstance(self.index, CloneNode))
...@@ -2386,7 +2404,10 @@ class IndexNode(ExprNode): ...@@ -2386,7 +2404,10 @@ class IndexNode(ExprNode):
if getting and self.type.is_pyobject: if getting and self.type.is_pyobject:
self.is_temp = True self.is_temp = True
if setting:
if setting and self.base.type.is_memoryviewslice:
self.type.writable_needed = True
elif setting:
if not self.base.entry.type.writable: if not self.base.entry.type.writable:
error(self.pos, "Writing to readonly buffer") error(self.pos, "Writing to readonly buffer")
else: else:
...@@ -3990,8 +4011,8 @@ class AttributeNode(ExprNode): ...@@ -3990,8 +4011,8 @@ class AttributeNode(ExprNode):
import MemoryView import MemoryView
MemoryView.put_assign_to_memviewslice(select_code, rhs.result(), self.type, MemoryView.put_assign_to_memviewslice(select_code, rhs.result(), self.type,
pos=self.pos, code=code) pos=self.pos, code=code)
if rhs.is_temp: #if rhs.is_temp:
code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type) # code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type)
if not self.type.is_memoryviewslice: if not self.type.is_memoryviewslice:
code.putln( code.putln(
"%s = %s;" % ( "%s = %s;" % (
...@@ -7647,46 +7668,14 @@ class CoerceToMemViewSliceNode(CoercionNode): ...@@ -7647,46 +7668,14 @@ class CoerceToMemViewSliceNode(CoercionNode):
self.type = dst_type self.type = dst_type
self.env = env self.env = env
self.is_temp = 1 self.is_temp = 1
self.arg = arg
def generate_result_code(self, code): def generate_result_code(self, code):
import MemoryView, Buffer self.type.create_from_py_utility_code(self.env)
memviewobj = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True) code.putln("%s = %s(%s);" % (self.result(),
buf_flag = MemoryView.get_buf_flag(self.type.axes) self.type.from_py_function,
code.putln("%s = (PyObject *) __pyx_memoryview_new(%s, %s);" % self.arg.py_result()))
(memviewobj, self.arg.py_result(), buf_flag))
code.putln(code.error_goto_if_PyErr(self.pos))
ndim = len(self.type.axes)
spec_int_arr = code.funcstate.allocate_temp(
PyrexTypes.c_array_type(PyrexTypes.c_int_type, ndim),
manage_ref=False)
specs_code = MemoryView.specs_to_code(self.type.axes)
for idx, cspec in enumerate(specs_code):
code.putln("%s[%d] = %s;" % (spec_int_arr, idx, cspec))
code.globalstate.use_utility_code(Buffer.acquire_utility_code)
code.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
dtype_typeinfo = Buffer.get_type_information_cname(code, self.type.dtype)
MemoryView.put_init_entry(self.result(), code)
code.putln("{")
code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" %
self.type.dtype.struct_nesting_depth())
result = self.result()
if self.type.is_c_contig:
c_or_f_flag = "__Pyx_IS_C_CONTIG"
elif self.type.is_f_contig:
c_or_f_flag = "__Pyx_IS_F_CONTIG"
else:
c_or_f_flag = "0"
code.putln(code.error_goto_if("-1 == __Pyx_ValidateAndInit_memviewslice("
"(struct __pyx_memoryview_obj *) %(memviewobj)s,"
" %(spec_int_arr)s, %(c_or_f_flag)s, %(ndim)d,"
" &%(dtype_typeinfo)s, __pyx_stack, &%(result)s)" % locals(), self.pos))
code.putln("}")
code.put_gotref(
code.as_pyobject("%s.memview" % self.result(), cython_memoryview_ptr_type))
code.funcstate.release_temp(memviewobj)
code.funcstate.release_temp(spec_int_arr)
class CastNode(CoercionNode): class CastNode(CoercionNode):
# Wrap a node in a C type cast. # Wrap a node in a C type cast.
......
...@@ -430,7 +430,7 @@ def create_default_resultobj(compilation_source, options): ...@@ -430,7 +430,7 @@ def create_default_resultobj(compilation_source, options):
def run_pipeline(source, options, full_module_name = None): def run_pipeline(source, options, full_module_name = None):
import Pipeline import Pipeline
# Set up context
context = options.create_context() context = options.create_context()
# Set up source object # Set up source object
...@@ -438,6 +438,7 @@ def run_pipeline(source, options, full_module_name = None): ...@@ -438,6 +438,7 @@ def run_pipeline(source, options, full_module_name = None):
abs_path = os.path.abspath(source) abs_path = os.path.abspath(source)
source_ext = os.path.splitext(source)[1] source_ext = os.path.splitext(source)[1]
full_module_name = full_module_name or context.extract_module_name(source, options) full_module_name = full_module_name or context.extract_module_name(source, options)
if options.relative_path_in_code_position_comments: if options.relative_path_in_code_position_comments:
rel_path = full_module_name.replace('.', os.sep) + source_ext rel_path = full_module_name.replace('.', os.sep) + source_ext
if not abs_path.endswith(rel_path): if not abs_path.endswith(rel_path):
...@@ -584,7 +585,8 @@ def compile_multiple(sources, options): ...@@ -584,7 +585,8 @@ def compile_multiple(sources, options):
a CompilationResultSet. Performs timestamp checking and/or recursion a CompilationResultSet. Performs timestamp checking and/or recursion
if these are specified in the options. if these are specified in the options.
""" """
context = options.create_context() # run_pipeline creates the context
# context = options.create_context()
sources = [os.path.abspath(source) for source in sources] sources = [os.path.abspath(source) for source in sources]
processed = set() processed = set()
results = CompilationResultSet() results = CompilationResultSet()
......
...@@ -46,17 +46,21 @@ _spec_to_const = { ...@@ -46,17 +46,21 @@ _spec_to_const = {
'follow' : MEMVIEW_FOLLOW, 'follow' : MEMVIEW_FOLLOW,
} }
_spec_to_abbrev = {
'direct' : 'd',
'ptr' : 'p',
'full' : 'f',
'contig' : 'c',
'strided' : 's',
'follow' : '_',
}
memview_name = u'memoryview' memview_name = u'memoryview'
memview_typeptr_cname = '__pyx_memoryview_type' memview_typeptr_cname = '__pyx_memoryview_type'
memview_objstruct_cname = '__pyx_memoryview_obj' memview_objstruct_cname = '__pyx_memoryview_obj'
memviewslice_cname = u'__Pyx_memviewslice' memviewslice_cname = u'__Pyx_memviewslice'
def specs_to_code(specs):
arr = []
for access, packing in specs:
arr.append("(%s | %s)" % (_spec_to_const[access],
_spec_to_const[packing]))
return arr
def put_init_entry(mv_cname, code): def put_init_entry(mv_cname, code):
code.putln("%s.data = NULL;" % mv_cname) code.putln("%s.data = NULL;" % mv_cname)
...@@ -70,7 +74,7 @@ def mangle_dtype_name(dtype): ...@@ -70,7 +74,7 @@ def mangle_dtype_name(dtype):
def axes_to_str(axes): def axes_to_str(axes):
return "".join([access[0].upper()+packing[0] for (access, packing) in axes]) return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code): def put_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code):
# import MemoryView # import MemoryView
assert rhs.type.is_memoryviewslice assert rhs.type.is_memoryviewslice
...@@ -80,32 +84,17 @@ def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_p ...@@ -80,32 +84,17 @@ def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_p
else: else:
rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False) rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False)
code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type))) code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type)))
code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
if not rhs.result_in_temp():
code.put_incref("%s.memview" % rhstmp, cython_memoryview_ptr_type)
if lhs_is_cglobal:
code.put_gotref("%s.memview" % lhs_result)
#XXX: this is here because self.lhs_of_first_assignment is not set correctly,
# once that is working this should take that flag into account.
# See NameNode.generate_assignment_code
code.put_xdecref("%s.memview" % lhs_result, cython_memoryview_ptr_type)
if lhs_is_cglobal:
code.put_giveref("%s.memview" % rhstmp)
code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
put_assign_to_memviewslice(lhs_result, rhstmp, lhs_type, put_assign_to_memviewslice(lhs_result, rhstmp, lhs_type,
lhs_pos, code=code) lhs_pos, code=code)
if rhs.result_in_temp() or not pretty_rhs:
code.putln("%s.memview = 0;" % rhstmp)
if not pretty_rhs: if not pretty_rhs:
code.funcstate.release_temp(rhstmp) code.funcstate.release_temp(rhstmp)
def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code): def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code):
code.put_xdecref_memoryviewslice(lhs_cname)
code.put_incref_memoryviewslice(rhs_cname)
code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname)) code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname))
code.putln("%s.data = %s.data;" % (lhs_cname, rhs_cname)) code.putln("%s.data = %s.data;" % (lhs_cname, rhs_cname))
...@@ -195,6 +184,45 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): ...@@ -195,6 +184,45 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
def get_buf_shapevars(self): def get_buf_shapevars(self):
return self._for_all_ndim("%s.shape[%d]") return self._for_all_ndim("%s.shape[%d]")
def generate_buffer_lookup_code(self, code, index_cnames):
bufp = self.buf_ptr
type_decl = self.type.dtype.declaration_code("")
for dim, (access, packing) in enumerate(self.type.axes):
shape = "%s.shape[%d]" % (self.cname, dim)
stride = "%s.strides[%d]" % (self.cname, dim)
suboffset = "%s.suboffsets[%d]" % (self.cname, dim)
index = index_cnames[dim]
if access == 'full' and packing in ('strided', 'follow'):
code.globalstate.use_utility_code(memviewslice_index_helpers)
bufp = ('__pyx_memviewslice_index_full(%s, %s, %s)' %
(bufp, index, stride, suboffset))
elif access == 'full' and packing == 'contig':
# We can skip stride multiplication with the cast
code.globalstate.use_utility_code(memviewslice_index_helpers)
bufp = '((char *) ((%s *) %s) + %s)' % (type_decl, bufp, index)
bufp = ('__pyx_memviewslice_index_full_contig(%s, %s)' %
(bufp, suboffset))
elif access == 'ptr' and packing in ('strided', 'follow'):
bufp = ("(*((char **) %s + %s * %s) + %s)" %
(bufp, index, stride, suboffset))
elif access == 'ptr' and packing == 'contig':
bufp = "(*((char **) %s) + %s)" % (bufp, suboffset)
elif access == 'direct' and packing in ('strided', 'follow'):
bufp = "(%s + %s * %s)" % (bufp, index, stride)
else:
assert (access, packing) == ('direct', 'contig'), (access, packing)
bufp = '((char *) (((%s *) %s) + %s))' % (type_decl, bufp, index)
bufp = '( /* dim=%d */ %s )' % (dim, bufp)
return "((%s *) %s)" % (type_decl, bufp)
def get_copy_func_name(to_memview): def get_copy_func_name(to_memview):
base = "__Pyx_BufferNew_%s_From_%s_%s" base = "__Pyx_BufferNew_%s_From_%s_%s"
...@@ -370,10 +398,10 @@ class CopyFuncUtilCode(object): ...@@ -370,10 +398,10 @@ class CopyFuncUtilCode(object):
if self.to_memview.is_c_contig: if self.to_memview.is_c_contig:
mode = 'c' mode = 'c'
contig_flag = 'PyBUF_C_CONTIGUOUS' contig_flag = memview_c_contiguous
elif self.to_memview.is_f_contig: elif self.to_memview.is_f_contig:
mode = 'fortran' mode = 'fortran'
contig_flag = "PyBUF_F_CONTIGUOUS" contig_flag = memview_f_contiguous
context = dict( context = dict(
copy_name=self.copy_func_name, copy_name=self.copy_func_name,
...@@ -735,11 +763,13 @@ class MemoryViewSliceTransform(CythonTransform): ...@@ -735,11 +763,13 @@ class MemoryViewSliceTransform(CythonTransform):
return node return node
def load_memview_cy_utility(util_code_name, **kwargs): def load_memview_cy_utility(util_code_name, context=None, **kwargs):
return CythonUtilityCode.load(util_code_name, "MemoryView.pyx", **kwargs) return CythonUtilityCode.load(util_code_name, "MemoryView.pyx",
context=context, **kwargs)
def load_memview_c_utility(util_code_name, **kwargs): def load_memview_c_utility(util_code_name, context=None, **kwargs):
return UtilityCode.load(util_code_name, "MemoryView_C.c", **kwargs) return UtilityCode.load(util_code_name, "MemoryView_C.c",
context=context, **kwargs)
context = { context = {
'memview_struct_name': memview_objstruct_cname, 'memview_struct_name': memview_objstruct_cname,
...@@ -753,6 +783,8 @@ memviewslice_declare_code = load_memview_c_utility( ...@@ -753,6 +783,8 @@ memviewslice_declare_code = load_memview_c_utility(
memviewslice_init_code = load_memview_c_utility( memviewslice_init_code = load_memview_c_utility(
"MemviewSliceInit", "MemviewSliceInit",
context={'BUF_MAX_NDIMS': Options.buffer_max_dims}, context=dict(context, BUF_MAX_NDIMS=Options.buffer_max_dims),
requires=[memviewslice_declare_code], requires=[memviewslice_declare_code],
) )
memviewslice_index_helpers = load_memview_c_utility("MemviewSliceIndex")
\ No newline at end of file
...@@ -79,6 +79,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -79,6 +79,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
scope.python_include_files) scope.python_include_files)
if merge_scope: if merge_scope:
# Ensure that we don't generate import code for these entries!
for entry in scope.c_class_entries:
entry.type.module_name = self.full_module_name
self.scope.merge_in(scope) self.scope.merge_in(scope)
def analyse_declarations(self, env): def analyse_declarations(self, env):
...@@ -345,7 +349,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -345,7 +349,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
self.generate_declarations_for_modules(env, modules, globalstate) self.generate_declarations_for_modules(env, modules, globalstate)
h_code.write('\n') h_code.write('\n')
for utilcode in env.utility_code_list: for utilcode in env.utility_code_list[:]:
globalstate.use_utility_code(utilcode) globalstate.use_utility_code(utilcode)
globalstate.finalize_main_c_code() globalstate.finalize_main_c_code()
...@@ -2255,7 +2259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -2255,7 +2259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
# Generate type import code for extern extension types # Generate type import code for extern extension types
# and type ready code for non-extern ones. # and type ready code for non-extern ones.
for entry in env.c_class_entries: for entry in env.c_class_entries:
if entry.visibility == 'extern': if entry.visibility == 'extern' and not entry.utility_code_definition:
self.generate_type_import_code(env, entry.type, entry.pos, code) self.generate_type_import_code(env, entry.type, entry.pos, code)
else: else:
self.generate_base_type_import_code(env, entry, code) self.generate_base_type_import_code(env, entry, code)
...@@ -2265,8 +2269,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -2265,8 +2269,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
def generate_base_type_import_code(self, env, entry, code): def generate_base_type_import_code(self, env, entry, code):
base_type = entry.type.base_type base_type = entry.type.base_type
if base_type and base_type.module_name != env.qualified_name \ if (base_type and base_type.module_name != env.qualified_name and not
and not base_type.is_builtin_type: base_type.is_builtin_type and not entry.utility_code_definition):
self.generate_type_import_code(env, base_type, self.pos, code) self.generate_type_import_code(env, base_type, self.pos, code)
def use_type_import_utility_code(self, env): def use_type_import_utility_code(self, env):
......
...@@ -1344,17 +1344,19 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1344,17 +1344,19 @@ class FuncDefNode(StatNode, BlockNode):
if not entry.in_closure: if not entry.in_closure:
code.put_var_declaration(entry) code.put_var_declaration(entry)
# Initialize the return variable __pyx_r
init = "" init = ""
if not self.return_type.is_void: if not self.return_type.is_void:
if self.return_type.is_pyobject: if self.return_type.is_pyobject:
init = " = NULL" init = " = NULL"
elif self.return_type.is_memoryviewslice:
init = "= {0, 0}"
code.putln( code.putln(
"%s%s;" % "%s%s;" %
(self.return_type.declaration_code(Naming.retval_cname), (self.return_type.declaration_code(Naming.retval_cname),
init)) init))
if self.return_type.is_memoryviewslice:
import MemoryView
MemoryView.put_init_entry(Naming.retval_cname, code)
tempvardecl_code = code.insertion_point() tempvardecl_code = code.insertion_point()
self.generate_keyword_list(code) self.generate_keyword_list(code)
...@@ -1431,15 +1433,18 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1431,15 +1433,18 @@ class FuncDefNode(StatNode, BlockNode):
if (acquire_gil or entry.assignments) and not entry.in_closure: if (acquire_gil or entry.assignments) and not entry.in_closure:
code.put_var_incref(entry) code.put_var_incref(entry)
if entry.type.is_memoryviewslice: if entry.type.is_memoryviewslice:
code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type) code.put_incref_memoryviewslice(entry.cname,
have_gil=not lenv.nogil)
#code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
# ----- Initialise local buffer auxiliary variables # ----- Initialise local buffer auxiliary variables
for entry in lenv.var_entries + lenv.arg_entries: for entry in lenv.var_entries + lenv.arg_entries:
if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used: if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
Buffer.put_init_vars(entry, code) Buffer.put_init_vars(entry, code)
# ----- Initialise local memoryviewslices # ----- Initialise local memoryviewslices
for entry in lenv.var_entries: for entry in lenv.var_entries:
if entry.type.is_memoryviewslice: if entry.visibility == "private" and not entry.used:
MemoryView.put_init_entry(entry.cname, code) continue
# ----- Check and convert arguments # ----- Check and convert arguments
self.generate_argument_type_tests(code) self.generate_argument_type_tests(code)
# ----- Acquire buffer arguments # ----- Acquire buffer arguments
...@@ -1544,7 +1549,8 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1544,7 +1549,8 @@ class FuncDefNode(StatNode, BlockNode):
if not entry.used or entry.in_closure: if not entry.used or entry.in_closure:
continue continue
if entry.type.is_memoryviewslice: if entry.type.is_memoryviewslice:
code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type) #code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
code.put_xdecref_memoryviewslice(entry.cname)
if entry.type.is_pyobject: if entry.type.is_pyobject:
code.put_var_decref(entry) code.put_var_decref(entry)
...@@ -1554,7 +1560,8 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1554,7 +1560,8 @@ class FuncDefNode(StatNode, BlockNode):
if (acquire_gil or entry.assignments) and not entry.in_closure: if (acquire_gil or entry.assignments) and not entry.in_closure:
code.put_var_decref(entry) code.put_var_decref(entry)
if entry.type.is_memoryviewslice: if entry.type.is_memoryviewslice:
code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type) code.put_xdecref_memoryviewslice(entry.cname)
#code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
if self.needs_closure: if self.needs_closure:
code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type) code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
...@@ -1567,8 +1574,9 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1567,8 +1574,9 @@ class FuncDefNode(StatNode, BlockNode):
err_val = default_retval err_val = default_retval
if self.return_type.is_pyobject: if self.return_type.is_pyobject:
code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname)) code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
elif self.return_type.is_memoryviewslice: #elif self.return_type.is_memoryviewslice:
code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type)) # code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type))
# code.put_xgiveref_memoryviewslice(Naming.retval_cname)
if self.entry.is_special and self.entry.name == "__hash__": if self.entry.is_special and self.entry.name == "__hash__":
# Returning -1 for __hash__ is supposed to signal an error # Returning -1 for __hash__ is supposed to signal an error
...@@ -4153,7 +4161,8 @@ class DelStatNode(StatNode): ...@@ -4153,7 +4161,8 @@ class DelStatNode(StatNode):
def analyse_expressions(self, env): def analyse_expressions(self, env):
for arg in self.args: for arg in self.args:
arg.analyse_target_expression(env, None) arg.analyse_target_expression(env, None)
if arg.type.is_pyobject: if arg.type.is_pyobject or (arg.is_name and
arg.type.is_memoryviewslice):
pass pass
elif arg.type.is_ptr and arg.type.base_type.is_cpp_class: elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
self.cpp_check(env) self.cpp_check(env)
...@@ -4172,7 +4181,7 @@ class DelStatNode(StatNode): ...@@ -4172,7 +4181,7 @@ class DelStatNode(StatNode):
def generate_execution_code(self, code): def generate_execution_code(self, code):
for arg in self.args: for arg in self.args:
if arg.type.is_pyobject: if arg.type.is_pyobject or arg.type.is_memoryviewslice:
arg.generate_deletion_code(code) arg.generate_deletion_code(code)
elif arg.type.is_ptr and arg.type.base_type.is_cpp_class: elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
arg.generate_result_code(code) arg.generate_result_code(code)
...@@ -4274,14 +4283,15 @@ class ReturnStatNode(StatNode): ...@@ -4274,14 +4283,15 @@ class ReturnStatNode(StatNode):
code.put_xdecref(Naming.retval_cname, code.put_xdecref(Naming.retval_cname,
self.return_type) self.return_type)
elif self.return_type.is_memoryviewslice: elif self.return_type.is_memoryviewslice:
code.put_xdecref("%s.memview" % Naming.retval_cname, code.put_xdecref_memoryviewslice(Naming.retval_cname)
self.return_type) #code.put_xdecref("%s.memview" % Naming.retval_cname,
# self.return_type)
if self.value: if self.value:
self.value.generate_evaluation_code(code) self.value.generate_evaluation_code(code)
if self.return_type.is_memoryviewslice: if self.return_type.is_memoryviewslice:
import MemoryView import MemoryView
MemoryView.gen_acquire_memoryviewslice(self.value, self.return_type, MemoryView.put_acquire_memoryviewslice(self.value, self.return_type,
False, Naming.retval_cname, None, code) False, Naming.retval_cname, None, code)
else: else:
self.value.make_owned_reference(code) self.value.make_owned_reference(code)
......
...@@ -2,12 +2,14 @@ ...@@ -2,12 +2,14 @@
# Cython/Python language types # Cython/Python language types
# #
from Code import UtilityCode from Code import UtilityCode, LazyUtilityCode
import StringEncoding import StringEncoding
import Naming import Naming
import copy import copy
from Errors import error from Errors import error
import cython
class BaseType(object): class BaseType(object):
# #
# Base class for all Cython types including pseudo-types. # Base class for all Cython types including pseudo-types.
...@@ -331,6 +333,15 @@ class MemoryViewSliceType(PyrexType): ...@@ -331,6 +333,15 @@ class MemoryViewSliceType(PyrexType):
has_attributes = 1 has_attributes = 1
scope = None scope = None
# These are specialcased in Defnode
from_py_function = None
to_py_function = None
exception_value = None
exception_check = None
utility_counter = 0
def __init__(self, base_dtype, axes): def __init__(self, base_dtype, axes):
''' '''
MemoryViewSliceType(base, axes) MemoryViewSliceType(base, axes)
...@@ -375,6 +386,7 @@ class MemoryViewSliceType(PyrexType): ...@@ -375,6 +386,7 @@ class MemoryViewSliceType(PyrexType):
assert not (self.is_c_contig and self.is_f_contig) assert not (self.is_c_contig and self.is_f_contig)
self.mode = MemoryView.get_mode(axes) self.mode = MemoryView.get_mode(axes)
self.writable_needed = False
def same_as_resolved_type(self, other_type): def same_as_resolved_type(self, other_type):
return ((other_type.is_memoryviewslice and return ((other_type.is_memoryviewslice and
...@@ -495,11 +507,68 @@ class MemoryViewSliceType(PyrexType): ...@@ -495,11 +507,68 @@ class MemoryViewSliceType(PyrexType):
def global_init_code(self, entry, code): def global_init_code(self, entry, code):
code.putln("%s.data = NULL;" % entry.cname) code.putln("%s.data = NULL;" % entry.cname)
code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False) code.putln("%s.memview = NULL;" % entry.cname)
#code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False)
def check_for_null_code(self, cname): def check_for_null_code(self, cname):
return cname + '.memview' return cname + '.memview'
def create_from_py_utility_code(self, env):
import MemoryView, Buffer, Code
# We don't have 'code', so use a LazyUtilityCode with a callback.
def lazy_utility_callback(code):
context['dtype_typeinfo'] = Buffer.get_type_information_cname(
code, self.dtype)
return Code.ContentHashingUtilityCode.load(
"ObjectToMemviewSlice", "MemoryView_C.c", context)
env.use_utility_code(Buffer.acquire_utility_code)
env.use_utility_code(MemoryView.memviewslice_init_code)
env.use_utility_code(LazyUtilityCode(lazy_utility_callback))
if self.is_c_contig:
c_or_f_flag = "__Pyx_IS_C_CONTIG"
elif self.is_f_contig:
c_or_f_flag = "__Pyx_IS_F_CONTIG"
else:
c_or_f_flag = "0"
# specializing through UtilityCode.specialize is not so useful as
# specialize on too many things to include in the function name
funcname = "__Pyx_PyObject_to_MemoryviewSlice_%d" % self.utility_counter
context = dict(
MemoryView.context,
buf_flag = MemoryView.get_buf_flag(self.axes),
ndim = self.ndim,
axes_specs = ', '.join(self.axes_specs_to_code()),
dtype_typedecl = self.dtype.declaration_code(""),
struct_nesting_depth = self.dtype.struct_nesting_depth(),
c_or_f_flag = c_or_f_flag,
funcname = funcname,
)
self.from_py_function = funcname
MemoryViewSliceType.utility_counter += 1
return True
def axes_specs_to_code(self):
"Return a list of code constants for each axis"
import MemoryView
d = MemoryView._spec_to_const
return ["(%s | %s)" % (d[a], d[p]) for a, p in self.axes]
def axes_specs_to_name(self):
"Return an abbreviated name for our axes"
import MemoryView
d = MemoryView._spec_to_abbrev
return "".join(["%s%s" % (d[a], d[p]) for a, p in self.axes])
def error_condition(self, result_code):
return "!%s.memview" % result_code
class BufferType(BaseType): class BufferType(BaseType):
# #
...@@ -2698,7 +2767,7 @@ cython_memoryview_type = CStructOrUnionType("__pyx_memoryview_obj", "struct", ...@@ -2698,7 +2767,7 @@ cython_memoryview_type = CStructOrUnionType("__pyx_memoryview_obj", "struct",
cython_memoryview_ptr_type = CPtrType(cython_memoryview_type) cython_memoryview_ptr_type = CPtrType(cython_memoryview_type)
memoryviewslice_type = CStructOrUnionType("__Pyx_memviewslice", "struct", memoryviewslice_type = CStructOrUnionType("memoryviewslice", "struct",
None, 1, "__Pyx_memviewslice") None, 1, "__Pyx_memviewslice")
error_type = ErrorType() error_type = ErrorType()
......
...@@ -382,6 +382,10 @@ class Scope(object): ...@@ -382,6 +382,10 @@ class Scope(object):
# entries[name] = entry # entries[name] = entry
if not shadow: if not shadow:
entries[name] = entry entries[name] = entry
if type.is_memoryviewslice:
entry.init = "{ 0, 0 }"
entry.scope = self entry.scope = self
entry.visibility = visibility entry.visibility = visibility
return entry return entry
......
...@@ -63,7 +63,8 @@ class CythonUtilityCode(Code.UtilityCodeBase): ...@@ -63,7 +63,8 @@ class CythonUtilityCode(Code.UtilityCodeBase):
is_cython_utility = True is_cython_utility = True
def __init__(self, impl, name="__pyxutil", prefix="", requires=None): def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
file=None):
# 1) We need to delay the parsing/processing, so that all modules can be # 1) We need to delay the parsing/processing, so that all modules can be
# imported without import loops # imported without import loops
# 2) The same utility code object can be used for multiple source files; # 2) The same utility code object can be used for multiple source files;
...@@ -72,6 +73,7 @@ class CythonUtilityCode(Code.UtilityCodeBase): ...@@ -72,6 +73,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
# Hence, delay any processing until later. # Hence, delay any processing until later.
self.impl = impl self.impl = impl
self.name = name self.name = name
self.file = file
self.prefix = prefix self.prefix = prefix
self.requires = requires or [] self.requires = requires or []
...@@ -113,10 +115,11 @@ class CythonUtilityCode(Code.UtilityCodeBase): ...@@ -113,10 +115,11 @@ class CythonUtilityCode(Code.UtilityCodeBase):
def put_code(self, output): def put_code(self, output):
pass pass
def declare_in_scope(self, dest_scope, used=False): def declare_in_scope(self, dest_scope, used=False, modname=None):
""" """
Declare all entries from the utility code in dest_scope. Code will only Declare all entries from the utility code in dest_scope. Code will only
be included for used entries. be included for used entries. If module_name is given, declare the
type entries with that name.
""" """
tree = self.get_tree(entries_only=True) tree = self.get_tree(entries_only=True)
...@@ -130,6 +133,10 @@ class CythonUtilityCode(Code.UtilityCodeBase): ...@@ -130,6 +133,10 @@ class CythonUtilityCode(Code.UtilityCodeBase):
entry.utility_code_definition = self entry.utility_code_definition = self
entry.used = used entry.used = used
if modname and entry.type.is_extension_type:
entry.qualified_name = modname
entry.type.module_name = modname
dest_scope.merge_in(tree.scope, merge_unused=True) dest_scope.merge_in(tree.scope, merge_unused=True)
tree.scope = dest_scope tree.scope = dest_scope
......
...@@ -10,7 +10,10 @@ cdef extern from "Python.h": ...@@ -10,7 +10,10 @@ cdef extern from "Python.h":
PyBUF_C_CONTIGUOUS, PyBUF_C_CONTIGUOUS,
PyBUF_F_CONTIGUOUS, PyBUF_F_CONTIGUOUS,
PyBUF_ANY_CONTIGUOUS PyBUF_ANY_CONTIGUOUS
PyBUF_FORMAT
cdef extern from *:
object __pyx_memoryview_new(object obj, int flags)
@cname("__pyx_array") @cname("__pyx_array")
cdef class array: cdef class array:
...@@ -105,7 +108,12 @@ cdef class array: ...@@ -105,7 +108,12 @@ cdef class array:
info.strides = self.strides info.strides = self.strides
info.suboffsets = NULL info.suboffsets = NULL
info.itemsize = self.itemsize info.itemsize = self.itemsize
if flags & PyBUF_FORMAT:
info.format = self.format info.format = self.format
else:
info.format = NULL
# we do not need to call releasebuffer # we do not need to call releasebuffer
info.obj = None info.obj = None
...@@ -130,6 +138,15 @@ cdef class array: ...@@ -130,6 +138,15 @@ cdef class array:
self.format = NULL self.format = NULL
self.itemsize = 0 self.itemsize = 0
def __getitem__(self, index):
view = __pyx_memoryview_new(self, PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT)
return view[index]
def __setitem__(self, index, value):
view = __pyx_memoryview_new(self, PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT)
view[index] = value
@cname("__pyx_array_new") @cname("__pyx_array_new")
cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode): cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode):
return array(shape, itemsize, format, mode.decode('ASCII')) return array(shape, itemsize, format, mode.decode('ASCII'))
...@@ -137,8 +154,10 @@ cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char * ...@@ -137,8 +154,10 @@ cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *
########## View.MemoryView ########## ########## View.MemoryView ##########
# from cpython cimport ... # from cpython cimport ...
cdef extern from "pythread.h": cdef extern from "Python.h":
int PyIndex_Check(object)
cdef extern from "pythread.h":
ctypedef void *PyThread_type_lock ctypedef void *PyThread_type_lock
PyThread_type_lock PyThread_allocate_lock() PyThread_type_lock PyThread_allocate_lock()
...@@ -150,6 +169,12 @@ cdef extern from *: ...@@ -150,6 +169,12 @@ cdef extern from *:
int __Pyx_GetBuffer(object, Py_buffer *, int) int __Pyx_GetBuffer(object, Py_buffer *, int)
void __Pyx_ReleaseBuffer(Py_buffer *) void __Pyx_ReleaseBuffer(Py_buffer *)
ctypedef struct {{memviewslice_name}}:
char *data
Py_ssize_t shape[{{max_dims}}]
Py_ssize_t strides[{{max_dims}}]
Py_ssize_t suboffsets[{{max_dims}}]
@cname('__pyx_MemviewEnum') @cname('__pyx_MemviewEnum')
cdef class Enum(object): cdef class Enum(object):
...@@ -173,23 +198,143 @@ cdef class memoryview(object): ...@@ -173,23 +198,143 @@ cdef class memoryview(object):
cdef object obj cdef object obj
cdef Py_buffer view cdef Py_buffer view
cdef PyThread_type_lock acqcnt_lock cdef PyThread_type_lock lock
cdef int acquisition_count cdef int acquisition_count
def __cinit__(memoryview self, object obj, int flags): def __cinit__(memoryview self, object obj, int flags):
self.obj = obj self.obj = obj
#self.acqcnt_lock = PyThread_allocate_lock()
#if self.acqcnt_lock == NULL:
# raise MemoryError
__Pyx_GetBuffer(obj, &self.view, flags) __Pyx_GetBuffer(obj, &self.view, flags)
self.lock = PyThread_allocate_lock()
if self.lock == NULL:
raise MemoryError
def __dealloc__(memoryview self): def __dealloc__(memoryview self):
#PyThread_free_lock(self.acqcnt_lock)
self.obj = None
__Pyx_ReleaseBuffer(&self.view) __Pyx_ReleaseBuffer(&self.view)
PyThread_free_lock(self.lock)
@cname('__pyx_memoryview_getitem')
def __getitem__(memoryview self, object index):
# cdef Py_ssize_t idx
cdef char *itemp = <char *> self.view.buf
cdef bytes bytesitem
cdef str fmt = self.view.format
import struct
try:
itemsize = struct.calcsize(fmt)
except struct.error:
raise TypeError("Unsupported format: %r" % fmt)
if index is Ellipsis:
return self
elif isinstance(index, slice):
if index == slice(None):
return self
raise NotImplementedError
else:
if not isinstance(index, tuple):
index = (index,)
tup = _unellipsify(index, self.view.ndim)
if len(tup) != self.view.ndim:
raise NotImplementedError(
"Expected %d indices (got %d)" %
(self.view.ndim, len(tup)))
for dim, idx in enumerate(tup):
_check_index(idx)
itemp = pybuffer_index(&self.view, itemp, idx, dim + 1)
# Do a manual and complete check here instead of this easy hack
bytesitem = itemp[:self.view.itemsize]
return struct.unpack(fmt, bytesitem)
@cname('__pyx_memoryviewslice')
cdef class _memoryviewslice(memoryview):
"Internal class for passing memory view slices to Python"
# We need this to keep our shape/strides/suboffset pointers valid
cdef {{memviewslice_name}} from_slice
# Restore the original Py_buffer before releasing
cdef Py_buffer orig_view
def __cinit__(self, object obj, int flags):
self.orig_view = self.view
def __dealloc__(self):
self.view = self.orig_view
@cname('__pyx_memoryview_new') @cname('__pyx_memoryview_new')
cdef memoryview memoryview_cwrapper(object o, int flags): cdef memoryview_cwrapper(object o, int flags):
return memoryview(o, flags) return memoryview(o, flags)
@cname('__pyx_memoryview_fromslice')
cdef memoryview memoryview_from_memview_cwrapper(memoryview m, int flags,
int new_ndim, {{memviewslice_name}} *memviewslice):
cdef _memoryviewslice result = _memoryviewslice(m.obj, flags)
result.from_slice = memviewslice[0]
result.view.shape = <Py_ssize_t *> (&result.from_slice.shape + new_ndim)
result.view.strides = <Py_ssize_t *> (&result.from_slice.strides + new_ndim)
result.view.suboffsets = <Py_ssize_t *> (&result.from_slice.suboffsets + new_ndim)
result.view.ndim = new_ndim
return result
cdef _check_index(index):
if not PyIndex_Check(index):
raise TypeError("Cannot index with %s" % type(index))
cdef tuple _unellipsify(tuple tup, int ndim):
if Ellipsis in tup:
result = []
for idx, item in enumerate(tup):
if item is Ellipsis:
result.extend([slice(None)] * (ndim - len(tup) + 1))
result.extend(tup[idx + 1:])
break
result.append(item)
return tuple(result)
return tup
@cname('__pyx_pybuffer_index')
cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, int dim) except NULL:
cdef Py_ssize_t shape, stride, suboffset = -1
cdef Py_ssize_t itemsize = view.itemsize
cdef char *resultp
if view.ndim == 0:
shape = view.len / itemsize
stride = itemsize
else:
shape = view.shape[dim]
stride = view.strides[dim]
if view.suboffsets != NULL:
suboffset = view.suboffsets[dim]
if index < 0:
index += view.shape[dim]
if index < 0:
raise IndexError("Out of bounds in dimension %d" % dim)
if index > shape:
raise IndexError("Out of bounds in dimension %d" % dim)
resultp = bufp + index * stride
if suboffset >= 0:
resultp = (<char **> resultp)[0] + suboffset
return resultp
...@@ -12,6 +12,9 @@ typedef struct { ...@@ -12,6 +12,9 @@ typedef struct {
Py_ssize_t suboffsets[{{max_dims}}]; Py_ssize_t suboffsets[{{max_dims}}];
} {{memviewslice_name}}; } {{memviewslice_name}};
/////////////// ObjectToMemviewSlice.proto ///////////////
{{# __Pyx_PyObject_to_MemoryviewSlice_<count> }}
static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *);
////////// MemviewSliceInit.proto ////////// ////////// MemviewSliceInit.proto //////////
...@@ -27,6 +30,7 @@ typedef struct { ...@@ -27,6 +30,7 @@ typedef struct {
#define __Pyx_IS_C_CONTIG 1 #define __Pyx_IS_C_CONTIG 1
#define __Pyx_IS_F_CONTIG 2 #define __Pyx_IS_F_CONTIG 2
/* #define __PYX_MEMSLICE_GETDATA(SLICE) ((char *) SLICE->memview->view->buf) */ /* #define __PYX_MEMSLICE_GETDATA(SLICE) ((char *) SLICE->memview->view->buf) */
static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview, static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview,
...@@ -38,6 +42,49 @@ static int __Pyx_init_memviewslice( ...@@ -38,6 +42,49 @@ static int __Pyx_init_memviewslice(
int ndim, int ndim,
__Pyx_memviewslice *memviewslice); __Pyx_memviewslice *memviewslice);
#define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__)
#define __PYX_XDEC_MEMVIEW(slice, have_gil) __Pyx_XDEC_MEMVIEW(slice, have_gil, __LINE__)
static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int);
static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int);
/////////////// MemviewSliceIndex.proto ///////////////
static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset);
static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset);
/////////////// ObjectToMemviewSlice ///////////////
{{#__Pyx_PyObject_to_MemoryviewSlice_<count>}}
static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
{{memviewslice_name}} result;
result.memview = NULL;
result.data = NULL;
struct __pyx_memoryview_obj *memview = \
(struct __pyx_memoryview_obj *) __pyx_memoryview_new(obj, {{buf_flag}});
__Pyx_BufFmt_StackElem stack[{{struct_nesting_depth}}];
int axes_specs[] = { {{axes_specs}} };
int retcode;
if (unlikely(!memview))
goto __pyx_fail;
retcode = __Pyx_ValidateAndInit_memviewslice(memview, axes_specs,
{{c_or_f_flag}}, {{ndim}}, &{{dtype_typeinfo}}, stack, &result);
if (unlikely(retcode == -1))
goto __pyx_fail;
memview->acquisition_count = 1;
return result;
__pyx_fail:
Py_XDECREF(memview);
result.memview = NULL;
result.data = NULL;
return result;
}
////////// MemviewSliceInit ////////// ////////// MemviewSliceInit //////////
static int __Pyx_ValidateAndInit_memviewslice( static int __Pyx_ValidateAndInit_memviewslice(
...@@ -203,8 +250,6 @@ static int __Pyx_init_memviewslice( ...@@ -203,8 +250,6 @@ static int __Pyx_init_memviewslice(
} }
} }
__Pyx_INCREF((PyObject *)memview);
__Pyx_GIVEREF((PyObject *)memview);
memviewslice->memview = memview; memviewslice->memview = memview;
memviewslice->data = (char *)buf->buf; memviewslice->data = (char *)buf->buf;
retval = 0; retval = 0;
...@@ -220,6 +265,62 @@ no_fail: ...@@ -220,6 +265,62 @@ no_fail:
return retval; return retval;
} }
static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
int have_gil, int lineno) {
int first_time;
struct {{memview_struct_name}} *memview = memslice->memview;
if (!memview) {
char msg[50];
snprintf(msg, 50, "memoryslice is not initialized (line %d)", lineno);
Py_FatalError(msg);
}
PyThread_acquire_lock(memview->lock, 1);
first_time = (memview->acquisition_count++ == 0);
PyThread_release_lock(memview->lock);
/* printf("INCREF %d: acquisition_count=%d, refcount=%d\n", lineno,
memview->acquisition_count, memview->ob_refcnt); */
if (first_time) {
if (have_gil) {
Py_INCREF((PyObject *) memview);
} else {
PyGILState_STATE _gilstate = PyGILState_Ensure();
Py_INCREF((PyObject *) memview);
PyGILState_Release(_gilstate);
}
}
}
static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
int have_gil, int lineno) {
int last_time;
struct {{memview_struct_name}} *memview = memslice->memview;
if (!memview) {
return;
}
PyThread_acquire_lock(memview->lock, 1);
last_time = (memview->acquisition_count-- == 1);
PyThread_release_lock(memview->lock);
/* printf("DECREF %d: acquisition_count=%d, refcount=%d\n", lineno,
memview->acquisition_count, memview->ob_refcnt); */
if (last_time) {
if (have_gil) {
Py_CLEAR(memview);
} else {
PyGILState_STATE _gilstate = PyGILState_Ensure();
Py_CLEAR(memview);
PyGILState_Release(_gilstate);
}
memslice->data = NULL;
}
}
////////// MemviewSliceCopyTemplate ////////// ////////// MemviewSliceCopyTemplate //////////
static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) { static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
...@@ -259,7 +360,8 @@ static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) { ...@@ -259,7 +360,8 @@ static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
} }
__Pyx_GOTREF(array_obj); __Pyx_GOTREF(array_obj);
memview_obj = __pyx_memoryview_new((PyObject *) array_obj, {{contig_flag}}); memview_obj = (struct __pyx_memoryview_obj *) __pyx_memoryview_new(
(PyObject *) array_obj, {{contig_flag}});
if (unlikely(!memview_obj)) { if (unlikely(!memview_obj)) {
goto fail; goto fail;
} }
...@@ -292,3 +394,21 @@ no_fail: ...@@ -292,3 +394,21 @@ no_fail:
} }
/////////////// MemviewSliceIndex ///////////////
static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset) {
bufp = bufp + idx * stride;
if (suboffset >= 0) {
bufp = *((char **) bufp) + suboffset;
}
return bufp;
}
/* The call has already done the indexing */
static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset) {
if (suboffset >= 0) {
bufp = *((char **) bufp) + suboffset;
}
return bufp;
}
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
# #
import os, sys, re, codecs import os, sys, re, codecs
from Cython import Tempita
def replace_suffix(path, newsuf): def replace_suffix(path, newsuf):
base, _ = os.path.splitext(path) base, _ = os.path.splitext(path)
...@@ -215,10 +216,3 @@ def long_literal(value): ...@@ -215,10 +216,3 @@ def long_literal(value):
if isinstance(value, basestring): if isinstance(value, basestring):
value = str_to_number(value) value = str_to_number(value)
return not -2**31 <= value < 2**31 return not -2**31 <= value < 2**31
def none_or_sub(s, data):
if s is None:
return s
else:
return s % data
This diff is collapsed.
...@@ -2,10 +2,8 @@ ...@@ -2,10 +2,8 @@
from __future__ import unicode_literals from __future__ import unicode_literals
# from cython cimport array from cython cimport array
# cimport cython.array as array
cimport cython as cy cimport cython as cy
# array = cython.array
def contiguity(): def contiguity():
''' '''
...@@ -69,3 +67,37 @@ def dont_allocate_buffer(): ...@@ -69,3 +67,37 @@ def dont_allocate_buffer():
cdef void callback(char *data): cdef void callback(char *data):
print "callback called %d" % <long> data print "callback called %d" % <long> data
cdef create_array(shape, mode):
cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode)
cdef int *data = <int *> result.data
cdef int i, j, cidx, fidx
for i in range(shape[0]):
for j in range(shape[1]):
cidx = i * shape[1] + j
fidx = i + j * shape[0]
if mode == 'fortran':
data[fidx] = cidx
else:
data[cidx] = cidx
return result
def test_cython_array():
"""
>>> test_cython_array()
98
61
98
61
"""
cdef int[:, ::1] carr = create_array((14, 10), 'c')
cdef int[::1, :] farr = create_array((14, 10), 'fortran')
print carr[9, 8]
print carr[6, 1]
print farr[9, 8]
print farr[6, 1]
This diff is collapsed.
cimport cython
from cython cimport array
from libc.stdlib cimport malloc, free
def create_array(shape, mode='c'):
cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode)
cdef int *data = <int *> result.data
cdef int i, j, value
for i in range(shape[0]):
for j in range(shape[1]):
value = i * shape[0] + j
if mode == 'fortran':
data[i + j * 10] = value
else:
data[value] = value
return result
def slice_contig_indexing():
"""
>>> print("disabled")
disabled
slice_contig_indexing()
98
61
98
61
"""
cdef int[:, ::1] carr = create_array((14, 10))
cdef int[::1, :] farr = create_array((10, 14), mode='fortran')
print carr[9, 8]
print carr[6, 1]
print farr[9, 8]
print farr[6, 1]
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment