initializedcheck directive + uninitialized memslice checking + memoryview...

initializedcheck directive + uninitialized memslice checking + memoryview indexing object<->dtype conversion

Cython/Compiler/ExprNodes.py

@@ -1422,6 +1422,7 @@ class NameNode(AtomicExprNode):
             self.result_ctype = py_object_type
 
     def analyse_types(self, env):
+        self.initialized_check = env.directives['initializedcheck']
         if self.entry is None:
             self.entry = env.lookup(self.name)
         if not self.entry:
@@ -1455,15 +1456,21 @@ class NameNode(AtomicExprNode):
         #print "Entry:", self.entry.__dict__ ###
         self.analyse_entry(env)
         entry = self.entry
+
         if entry.is_declared_generic:
             self.result_ctype = py_object_type
+
         if entry.is_pyglobal or entry.is_builtin:
             if entry.is_builtin and entry.is_const:
                 self.is_temp = 0
             else:
                 self.is_temp = 1
                 env.use_utility_code(get_name_interned_utility_code)
+
             self.is_used_as_rvalue = 1
+        elif entry.type.is_memoryviewslice:
+            self.is_temp = False
+            self.is_used_as_rvalue = True
 
     def nogil_check(self, env):
         self.nogil = True
@@ -1474,6 +1481,10 @@ class NameNode(AtomicExprNode):
                     self.gil_error()
             elif entry.is_pyglobal:
                 self.gil_error()
+            elif self.entry.type.is_memoryviewslice:
+                if self.cf_is_null or self.cf_maybe_null:
+                    import MemoryView
+                    MemoryView.err_if_nogil_initialized_check(self.pos, env)
 
     gil_message = "Accessing Python global or builtin"
 
@@ -1614,10 +1625,14 @@ class NameNode(AtomicExprNode):
             code.put_gotref(self.py_result())
 
         elif entry.is_local or entry.in_closure or entry.from_closure:
-            if entry.type.check_for_null_code(entry.cname):
-                if (self.cf_maybe_null or self.cf_is_null) \
-                       and not self.allow_null:
-                    code.put_error_if_unbound(self.pos, entry)
+            # Raise UnboundLocalError for objects and memoryviewslices
+            raise_unbound = (
+                (self.cf_maybe_null or self.cf_is_null) and not self.allow_null)
+            null_code = entry.type.check_for_null_code(entry.cname)
+            memslice_check = entry.type.is_memoryviewslice and self.initialized_check
+
+            if null_code and raise_unbound and (entry.type.is_pyobject or memslice_check):
+                code.put_error_if_unbound(self.pos, entry)
 
     def generate_assignment_code(self, rhs, code):
         #print "NameNode.generate_assignment_code:", self.name ###
@@ -2401,18 +2416,23 @@ class IndexNode(ExprNode):
                     if not x.type.is_int:
                         buffer_access = False
 
-            if buffer_access:
+            if buffer_access and not self.base.type.is_memoryviewslice:
                 assert hasattr(self.base, "entry") # Must be a NameNode-like node
 
         # On cloning, indices is cloned. Otherwise, unpack index into indices
         assert not (buffer_access and isinstance(self.index, CloneNode))
 
+        self.nogil = env.nogil
+
         if buffer_access:
+            if self.base.type.is_memoryviewslice and not self.base.is_name:
+                self.base = self.base.coerce_to_temp(env)
+
             self.indices = indices
             self.index = None
             self.type = self.base.type.dtype
             self.is_buffer_access = True
-            self.buffer_type = self.base.entry.type
+            self.buffer_type = self.base.type #self.base.entry.type
 
             if getting and self.type.is_pyobject:
                 self.is_temp = True
@@ -2734,9 +2754,14 @@ class IndexNode(ExprNode):
 
         # Generate buffer access code using these temps
         import Buffer, MemoryView
-        # The above could happen because child_attrs is wrong somewhere so that
-        # options are not propagated.
-        entry = self.base.entry
+
+        if self.base.is_name:
+            entry = self.base.entry
+        else:
+            assert self.base.is_temp
+            cname = self.base.result()
+            entry = Symtab.Entry(cname, cname, self.base.type, self.base.pos)
+
         if entry.type.is_buffer:
             buffer_entry = Buffer.BufferEntry(entry)
             negative_indices = entry.type.negative_indices
@@ -3345,20 +3370,27 @@ class SimpleCallNode(CallNode):
                     if i > 0 or i == 1 and self.self is not None: # skip first arg
                         warning(arg.pos, "Argument evaluation order in C function call is undefined and may not be as expected", 0)
                         break
+
         # Calc result type and code fragment
         if isinstance(self.function, NewExprNode):
             self.type = PyrexTypes.CPtrType(self.function.class_type)
         else:
             self.type = func_type.return_type
+
         if self.function.is_name or self.function.is_attribute:
             if self.function.entry and self.function.entry.utility_code:
                 self.is_temp = 1 # currently doesn't work for self.calculate_result_code()
+
         if self.type.is_pyobject:
             self.result_ctype = py_object_type
             self.is_temp = 1
         elif func_type.exception_value is not None \
                  or func_type.exception_check:
             self.is_temp = 1
+        elif self.type.is_memoryviewslice:
+            self.is_temp = 1
+            # func_type.exception_check = True
+
         # Called in 'nogil' context?
         self.nogil = env.nogil
         if (self.nogil and
@@ -3436,6 +3468,9 @@ class SimpleCallNode(CallNode):
             exc_checks = []
             if self.type.is_pyobject and self.is_temp:
                 exc_checks.append("!%s" % self.result())
+            elif self.type.is_memoryviewslice:
+                assert self.is_temp
+                exc_checks.append(self.type.error_condition(self.result()))
             else:
                 exc_val = func_type.exception_value
                 exc_check = func_type.exception_check
@@ -3758,6 +3793,7 @@ class AttributeNode(ExprNode):
         self.analyse_types(env, target = 1)
 
     def analyse_types(self, env, target = 0):
+        self.initialized_check = env.directives['initializedcheck']
         if self.analyse_as_cimported_attribute(env, target):
             self.entry.used = True
         elif not target and self.analyse_as_unbound_cmethod(env):
@@ -3862,6 +3898,8 @@ class AttributeNode(ExprNode):
                 self.result_ctype = py_object_type
         elif target and self.obj.type.is_builtin_type:
             error(self.pos, "Assignment to an immutable object field")
+        #elif self.type.is_memoryviewslice and not target:
+        #    self.is_temp = True
 
     def analyse_attribute(self, env, obj_type = None):
         # Look up attribute and set self.type and self.member.
@@ -3933,6 +3971,8 @@ class AttributeNode(ExprNode):
     def nogil_check(self, env):
         if self.is_py_attr:
             self.gil_error()
+        import MemoryView
+        MemoryView.err_if_nogil_initialized_check(self.pos, env, 'attribute')
 
     gil_message = "Accessing Python attribute"
 
@@ -3987,6 +4027,16 @@ class AttributeNode(ExprNode):
                     code.intern_identifier(self.attribute),
                     code.error_goto_if_null(self.result(), self.pos)))
             code.put_gotref(self.py_result())
+        elif self.type.is_memoryviewslice:
+            if self.initialized_check:
+                code.putln(textwrap.dedent('''
+                    if (unlikely(!%s.memview)) {
+                        PyErr_SetString(PyExc_AttributeError,
+                                        "Memoryview is not initialized");
+                        %s
+                    }''' % (self.result(), code.error_goto(self.pos))))
+            #code.putln("%s = %s;" % (self.result(),
+            #                         self.calculate_result_code()))
         else:
             # result_code contains what is needed, but we may need to insert
             # a check and raise an exception
@@ -7860,6 +7910,8 @@ class CoerceToPyTypeNode(CoercionNode):
             # FIXME: check that the target type and the resulting type are compatible
             pass
 
+        self.env = env
+
     gil_message = "Converting to Python object"
 
     def may_be_none(self):
@@ -7887,7 +7939,7 @@ class CoerceToPyTypeNode(CoercionNode):
 
     def generate_result_code(self, code):
         if self.arg.type.is_memoryviewslice:
-            funccall = self.arg.type.get_to_py_function(self.arg)
+            funccall = self.arg.type.get_to_py_function(self.env, self.arg)
         else:
             funccall = "%s(%s)" % (self.arg.type.to_py_function,
                                    self.arg.result())

Cython/Compiler/MemoryView.py

-from Errors import CompileError
+from Errors import CompileError, error
 import ExprNodes
 from ExprNodes import IntNode, NoneNode, IntBinopNode, NameNode, AttributeNode
 from Visitor import CythonTransform
@@ -17,6 +17,12 @@ BOTH_CF_ERR = "Cannot specify an array that is both C and Fortran contiguous."
 INVALID_ERR = "Invalid axis specification."
 EXPR_ERR = "no expressions allowed in axis spec, only names and literals."
 CF_ERR = "Invalid axis specification for a C/Fortran contiguous array."
+ERR_UNINITIALIZED = ("Cannot check if memoryview %s is initialized without the "
+                     "GIL, consider using initializedcheck(False)")
+
+def err_if_nogil_initialized_check(pos, env, name='variable'):
+    if env.nogil and env.directives['initializedcheck']:
+        error(pos, ERR_UNINITIALIZED % name)
 
 def concat_flags(*flags):
     return "(%s)" % "|".join(flags)
@@ -85,7 +91,8 @@ def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code,
         rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False)
         code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type)))
 
-    code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
+    # Allow uninitialized assignment
+    #code.putln(code.put_error_if_unbound(lhs_pos, rhs.entry))
     put_assign_to_memviewslice(lhs_cname, rhstmp, lhs_type, code, incref_rhs)
 
     if not pretty_rhs:
@@ -739,9 +746,15 @@ def _resolve_AttributeNode(env, node):
     modnames = path[:-1]
     # must be at least 1 module name, o/w not an AttributeNode.
     assert modnames
-    scope = env.lookup(modnames[0]).as_module
-    for modname in modnames[1:]:
-        scope = scope.lookup(modname).as_module
+
+    scope = env
+    for modname in modnames:
+        mod = scope.lookup(modname)
+        if not mod:
+            raise CompileError(
+                    node.pos, "undeclared name not builtin: %s" % modname)
+        scope = mod.as_module
+
     return scope.lookup(path[-1])
 
 def load_memview_cy_utility(util_code_name, context=None, **kwargs):

Cython/Compiler/Nodes.py

@@ -10,7 +10,7 @@ cython.declare(sys=object, os=object, time=object, copy=object,
                CppClassScope=object, UtilityCode=object, EncodedString=object,
                absolute_path_length=cython.Py_ssize_t)
 
-import sys, os, time, copy
+import sys, os, time, copy, textwrap
 
 import Builtin
 from Errors import error, warning, InternalError, CompileError
@@ -1501,7 +1501,12 @@ class FuncDefNode(StatNode, BlockNode):
                 #    code.put_xdecref_memoryviewslice(entry.cname)
                 code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}")
 
-            err_val = self.error_value()
+            if self.return_type.is_memoryviewslice:
+                MemoryView.put_init_entry(Naming.retval_cname, code)
+                err_val = Naming.retval_cname
+            else:
+                err_val = self.error_value()
+
             exc_check = self.caller_will_check_exceptions()
             if err_val is not None or exc_check:
                 # TODO: Fix exception tracing (though currently unused by cProfile).
@@ -1552,6 +1557,26 @@ class FuncDefNode(StatNode, BlockNode):
                 Buffer.put_release_buffer_code(code, entry)
         if is_getbuffer_slot:
             self.getbuffer_normal_cleanup(code)
+
+        if self.return_type.is_memoryviewslice:
+            # See if our return value is uninitialized on non-error return
+            # import MemoryView
+            # MemoryView.err_if_nogil_initialized_check(self.pos, env)
+            cond = code.unlikely(self.return_type.error_condition(
+                                                    Naming.retval_cname))
+            code.putln(
+                'if (%s) {' % cond)
+            if env.nogil:
+                code.put_ensure_gil()
+            code.putln(
+                    'PyErr_SetString('
+                        'PyExc_TypeError,'
+                        '"Memoryview return value is not initialized");')
+            if env.nogil:
+                code.put_release_ensured_gil()
+            code.putln(
+                '}')
+
         # ----- Return cleanup for both error and no-error return
         code.put_label(code.return_from_error_cleanup_label)
 
@@ -4312,8 +4337,6 @@ class ReturnStatNode(StatNode):
                              self.return_type)
         elif self.return_type.is_memoryviewslice:
             code.put_xdecref_memoryviewslice(Naming.retval_cname)
-            #code.put_xdecref("%s.memview" % Naming.retval_cname,
-            #        self.return_type)
 
         if self.value:
             self.value.generate_evaluation_code(code)
@@ -7042,7 +7065,12 @@ class CnameDecoratorNode(StatNode):
 
             for name, entry in scope.entries.iteritems():
                 if entry.func_cname:
-                    entry.func_cname = '%s_%s' % (self.cname, entry.cname)
+                    cname = entry.cname
+                    if '.' in cname:
+                        # remove __pyx_base from func_cname
+                        cname = cname.split('.')[-1]
+
+                    entry.func_cname = '%s_%s' % (self.cname, cname)
 
     def analyse_expressions(self, env):
         self.node.analyse_expressions(env)

Cython/Compiler/Options.py

@@ -75,6 +75,7 @@ buffer_max_dims = 32
 directive_defaults = {
     'boundscheck' : True,
     'nonecheck' : False,
+    'initializedcheck' : True,
     'embedsignature' : False,
     'locals' : {},
     'auto_cpdef': False,

Cython/Compiler/PyrexTypes.py

@@ -387,6 +387,8 @@ class MemoryViewSliceType(PyrexType):
         self.mode = MemoryView.get_mode(axes)
         self.writable_needed = False
 
+        self.dtype_name = MemoryView.mangle_dtype_name(self.dtype)
+
     def same_as_resolved_type(self, other_type):
         return ((other_type.is_memoryviewslice and
             self.dtype.same_as(other_type.dtype) and
@@ -500,9 +502,7 @@ class MemoryViewSliceType(PyrexType):
         return True
 
     def specialization_suffix(self):
-        import MemoryView
-        dtype_name = MemoryView.mangle_dtype_name(self.dtype)
-        return "%s_%s" % (self.axes_to_name(), dtype_name)
+        return "%s_%s" % (self.axes_to_name(), self.dtype_name)
 
     #def global_init_code(self, entry, code):
     #    code.putln("%s.data = NULL;" % entry.cname)
@@ -552,9 +552,49 @@ class MemoryViewSliceType(PyrexType):
     def create_to_py_utility_code(self, env):
         return True
 
-    def get_to_py_function(self, obj):
-        return "__pyx_memoryview_fromslice(&%s, %s.memview->obj, %s, %s);" % (
-                       obj.result(), obj.result(), self.flags, self.ndim)
+    def get_to_py_function(self, env, obj):
+        to_py_func, from_py_func = self.dtype_object_conversion_funcs(env)
+        to_py_func = "(PyObject *(*)(char *)) " + to_py_func
+        from_py_func = "(int (*)(char *, PyObject *)) " + from_py_func
+
+        tup = (obj.result(), obj.result(), self.flags, self.ndim,
+               to_py_func, from_py_func)
+        return ("__pyx_memoryview_fromslice(&%s, %s.memview->obj, "
+                                            "%s, %s, %s, %s);" % tup)
+
+    def dtype_object_conversion_funcs(self, env):
+        import MemoryView, Code
+
+        get_function = "__pyx_memview_get_%s" % self.dtype_name
+        set_function = "__pyx_memview_set_%s" % self.dtype_name
+
+        context = dict(
+            get_function = get_function,
+            set_function = set_function,
+        )
+
+        if self.dtype.is_pyobject:
+            utility_name = "MemviewObjectToObject"
+        else:
+            if not (self.dtype.create_to_py_utility_code(env) and
+                    self.dtype.create_from_py_utility_code(env)):
+                print "cannot convert %s" % self.dtype
+                return "NULL", "NULL"
+
+            utility_name = "MemviewDtypeToObject"
+            error_condition = (self.dtype.error_condition('value') or
+                               'PyErr_Occurred()')
+            context.update(
+                to_py_function = self.dtype.to_py_function,
+                from_py_function = self.dtype.from_py_function,
+                dtype = self.dtype.declaration_code(""),
+                error_condition = error_condition,
+            )
+
+        utility = Code.ContentHashingUtilityCode.load(
+                        utility_name, "MemoryView_C.c", context=context)
+        env.use_utility_code(utility)
+        return get_function, set_function
 
     def axes_to_code(self):
         "Return a list of code constants for each axis"

Cython/Compiler/UtilityCode.py

@@ -64,7 +64,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
     is_cython_utility = True
 
     def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
-                 file=None):
+                 file=None, from_scope=None):
         # 1) We need to delay the parsing/processing, so that all modules can be
         #    imported without import loops
         # 2) The same utility code object can be used for multiple source files;
@@ -76,6 +76,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
         self.file = file
         self.prefix = prefix
         self.requires = requires or []
+        self.from_scope = from_scope
 
     def get_tree(self, entries_only=False):
         from AnalysedTreeTransforms import AutoTestDictTransform
@@ -108,6 +109,15 @@ class CythonUtilityCode(Code.UtilityCodeBase):
         pipeline = Pipeline.insert_into_pipeline(pipeline, transform,
                                                  before=before)
 
+        if self.from_scope:
+            def scope_transform(module_node):
+                module_node.scope.merge_in(self.from_scope)
+                return module_node
+
+            transform = ParseTreeTransforms.AnalyseDeclarationsTransform
+            pipeline = Pipeline.insert_into_pipeline(pipeline, transform,
+                                                     before=scope_transform)
+
         (err, tree) = Pipeline.run_pipeline(pipeline, tree)
         assert not err, err
         return tree

Cython/Utility/MemoryView.pyx

@@ -217,18 +217,10 @@ cdef class memoryview(object):
         if self.lock != NULL:
             PyThread_free_lock(self.lock)
 
-    @cname('__pyx_memoryview_getitem')
-    def __getitem__(memoryview self, object index):
-        # cdef Py_ssize_t idx
-        cdef char *itemp = <char *> self.view.buf
-        cdef bytes bytesitem
-        cdef str fmt = self.view.format
-
-        import struct
-        try:
-            itemsize = struct.calcsize(fmt)
-        except struct.error:
-            raise TypeError("Unsupported format: %r" % fmt)
+    cdef char *get_item_pointer(memoryview self, object index) except NULL:
+        cdef Py_ssize_t dim
+        cdef Py_buffer view = self.view
+        cdef char *itemp = <char *> view.buf
 
         if index is Ellipsis:
             return self
@@ -252,17 +244,50 @@ cdef class memoryview(object):
 
             for dim, idx in enumerate(tup):
                 _check_index(idx)
-                itemp = pybuffer_index(&self.view, itemp, idx, dim + 1)
+                itemp = pybuffer_index(&self.view, itemp, idx, dim)
+
+        return itemp
 
+    @cname('__pyx_memoryview_getitem')
+    def __getitem__(memoryview self, object index):
+        cdef char *itemp = self.get_item_pointer(index)
+        return self.convert_item_to_object(itemp)
+
+    @cname('__pyx_memoryview_setitem')
+    def __setitem__(memoryview self, object index, object value):
+        cdef char *itemp = self.get_item_pointer(index)
+        self.assign_item_from_object(itemp, value)
+
+    cdef convert_item_to_object(self, char *itemp):
+        """Only used if instantiated manually by the user, or if Cython doesn't
+        know how to convert the type"""
+        import struct
+        cdef bytes bytesitem
         # Do a manual and complete check here instead of this easy hack
         bytesitem = itemp[:self.view.itemsize]
-        return struct.unpack(fmt, bytesitem)
+        return struct.unpack(self.view.format, bytesitem)
+
+    cdef assign_item_from_object(self, char *itemp, object value):
+        """Only used if instantiated manually by the user, or if Cython doesn't
+        know how to convert the type"""
+        import struct
+        cdef char c
+        cdef bytes bytesvalue
+        cdef Py_ssize_t i
+
+        if isinstance(value, tuple):
+            bytesvalue = struct.pack(self.view.format, *value)
+        else:
+            bytesvalue = struct.pack(self.view.format, value)
+
+        for i, c in enumerate(bytesvalue):
+            itemp[i] = c
 
     def __repr__(self):
-        return "<MemoryView of %s at 0x%x>" % (self.obj.__class__.__name__, id(self))
+        return "<MemoryView of %r at 0x%x>" % (self.obj.__class__.__name__, id(self))
 
     def __str__(self):
-        return "<MemoryView of %r at 0x%x>" % (self.obj, id(self))
+        return "<MemoryView of %r object>" % (self.obj.__class__.__name__,)
 
 
 @cname('__pyx_memoryviewslice')
@@ -274,28 +299,47 @@ cdef class _memoryviewslice(memoryview):
     # Restore the original Py_buffer before releasing
     cdef Py_buffer orig_view
 
+    cdef object (*to_object_func)(char *)
+    cdef int (*to_dtype_func)(char *, object) except 0
+
     def __cinit__(self, object obj, int flags):
         self.orig_view = self.view
 
     def __dealloc__(self):
         self.view = self.orig_view
 
+    cdef convert_item_to_object(self, char *itemp):
+        if self.to_object_func != NULL:
+            self.to_object_func(itemp)
+        else:
+            memoryview.convert_item_to_object(self, itemp)
+
+    cdef assign_item_from_object(self, char *itemp, object value):
+        if self.to_dtype_func != NULL:
+            self.to_dtype_func(itemp, value)
+        else:
+            memoryview.assign_item_from_object(self, itemp, value)
+
 
 @cname('__pyx_memoryview_new')
 cdef memoryview_cwrapper(object o, int flags):
     return memoryview(o, flags)
 
 @cname('__pyx_memoryview_fromslice')
-cdef memoryview_from_memview_cwrapper({{memviewslice_name}} *memviewslice,
-                                object orig_obj, int flags, int new_ndim):
+cdef memoryview_from_memslice_cwrapper(
+            {{memviewslice_name}} *memviewslice, object orig_obj, int flags, int cur_ndim,
+             object (*to_object_func)(char *), int (*to_dtype_func)(char *, object)):
     cdef _memoryviewslice result = _memoryviewslice(orig_obj, flags)
+    cdef int new_ndim = result.view.ndim - cur_ndim
 
     result.from_slice = memviewslice[0]
 
     result.view.shape = <Py_ssize_t *> (&result.from_slice.shape + new_ndim)
     result.view.strides = <Py_ssize_t *> (&result.from_slice.strides + new_ndim)
     result.view.suboffsets = <Py_ssize_t *> (&result.from_slice.suboffsets + new_ndim)
-    result.view.ndim = new_ndim
+    result.view.ndim = cur_ndim
+
+    result.to_object_func = to_object_func
 
     return result
 
@@ -336,14 +380,14 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, int dim
     if index < 0:
         index += view.shape[dim]
         if index < 0:
-            raise IndexError("Out of bounds in dimension %d" % dim)
+            raise IndexError("Out of bounds on buffer access (axis %d)" % dim)
 
     if index > shape:
-        raise IndexError("Out of bounds in dimension %d" % dim)
+        raise IndexError("Out of bounds on buffer access (axis %d)" % dim)
 
     resultp = bufp + index * stride
-
     if suboffset >= 0:
         resultp = (<char **> resultp)[0] + suboffset
 
     return resultp
+

Cython/Utility/MemoryView_C.c

@@ -49,10 +49,7 @@ static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int);
 static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int);
 
 /////////////// MemviewSliceIndex.proto ///////////////
-
-static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset);
-static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset);
-
+static CYTHON_INLINE char *__pyx_memviewslice_index_full(const char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset);
 
 /////////////// ObjectToMemviewSlice ///////////////
 
@@ -173,10 +170,6 @@ static int __Pyx_ValidateAndInit_memviewslice(
             }
         }
 
-        if (spec & (__Pyx_MEMVIEW_PTR|__Pyx_MEMVIEW_FULL) && !buf->suboffsets) {
-            memviewslice->suboffsets[i] = -1;
-        }
-
         if (spec & __Pyx_MEMVIEW_PTR) {
             if (buf->suboffsets && buf->suboffsets[i] < 0) {
                 PyErr_Format(PyExc_ValueError,
@@ -238,16 +231,16 @@ static int __Pyx_init_memviewslice(
         PyErr_SetString(PyExc_ValueError,
             "buf is NULL.");
         goto fail;
-    } else if(memviewslice->memview || memviewslice->data) {
+    } else if (memviewslice->memview || memviewslice->data) {
         PyErr_SetString(PyExc_ValueError,
             "memviewslice is already initialized!");
         goto fail;
     }
 
-    for(i=0; i<ndim; i++) {
+    for (i = 0; i < ndim; i++) {
         memviewslice->strides[i] = buf->strides[i];
         memviewslice->shape[i]   = buf->shape[i];
-        if(buf->suboffsets) {
+        if (buf->suboffsets) {
             memviewslice->suboffsets[i] = buf->suboffsets[i];
         } else {
             memviewslice->suboffsets[i] = -1;
@@ -294,15 +287,16 @@ static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
     int first_time;
     struct {{memview_struct_name}} *memview = memslice->memview;
     if (!memview)
-        __pyx_fatalerror("memoryslice is not initialized (line %d)", lineno);
+        return; /* allow uninitialized memoryview assignment */
+        /* __pyx_fatalerror("memoryslice is not initialized (line %d)", lineno); */
 
     if (memview->acquisition_count <= 0)
         __pyx_fatalerror("Acquisition count is %d (line %d)",
                          memview->acquisition_count, lineno);
 
-    PyThread_acquire_lock(memview->lock, 1);
+    //PyThread_acquire_lock(memview->lock, 1);
     first_time = (memview->acquisition_count++ == 0);
-    PyThread_release_lock(memview->lock);
+    //PyThread_release_lock(memview->lock);
 
     if (first_time) {
         if (have_gil) {
@@ -327,9 +321,9 @@ static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
         __pyx_fatalerror("Acquisition count is %d (line %d)",
                          memview->acquisition_count, lineno);
 
-    PyThread_acquire_lock(memview->lock, 1);
+    //PyThread_acquire_lock(memview->lock, 1);
     last_time = (memview->acquisition_count-- == 1);
-    PyThread_release_lock(memview->lock);
+    //PyThread_release_lock(memview->lock);
 
     if (last_time) {
         if (have_gil) {
@@ -418,10 +412,46 @@ no_fail:
 
 /////////////// MemviewSliceIndex ///////////////
 
-static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset) {
+static CYTHON_INLINE char *__pyx_memviewslice_index_full(const char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset) {
     bufp = bufp + idx * stride;
     if (suboffset >= 0) {
         bufp = *((char **) bufp) + suboffset;
     }
     return bufp;
 }
+
+/////////////// MemviewDtypeToObject.proto ///////////////
+PyObject *{{get_function}}(const char *itemp); /* proto */
+int {{set_function}}(const char *itemp, PyObject *obj); /* proto */
+
+/////////////// MemviewDtypeToObject ///////////////
+{{#__pyx_memview_<dtype_name>_to_object}}
+
+PyObject *{{get_function}}(const char *itemp) {
+    return (PyObject *) {{to_py_function}}(*({{dtype}} *) itemp);
+}
+
+int {{set_function}}(const char *itemp, PyObject *obj) {
+    {{dtype}} value = {{from_py_function}}(obj);
+    if ({{error_condition}})
+        return 0;
+    *({{dtype}} *) itemp = value;
+    return 1;
+}
+
+/////////////// MemviewObjectToObject.proto ///////////////
+PyObject *{{get_function}}(const char *itemp); /* proto */
+int {{set_function}}(const char *itemp, PyObject *obj); /* proto */
+
+/////////////// MemviewObjectToObject ///////////////
+PyObject *{{get_function}}(const char *itemp) {
+    PyObject *result = *(PyObject **) itemp;
+    Py_INCREF(result);
+    return result;
+}
+
+int {{set_function}}(const char *itemp, PyObject *obj) {
+    Py_INCREF(obj);
+    *(PyObject **) itemp = obj;
+    return 1;
+}

tests/run/memoryview.pyx

@@ -9,7 +9,13 @@ u'''
 
 from cython.view cimport memoryview
 from cython cimport array, PyBUF_C_CONTIGUOUS
+from cython cimport view
 
+include "mockbuffers.pxi"
+
+#
+### Test for some coercions
+#
 def init_obj():
     return 3
 
@@ -38,14 +44,14 @@ def g():
     obj = init_obj()
     mview = array((10,), itemsize=sizeof(int), format='i')
 
-cdef class Foo:
+cdef class ExtClass(object):
     cdef int[::1] mview
 
     def __init__(self):
         self.mview = array((10,), itemsize=sizeof(int), format='i')
         self.mview = array((10,), itemsize=sizeof(int), format='i')
 
-class pyfoo:
+class PyClass(object):
 
     def __init__(self):
         self.mview = array((10,), itemsize=sizeof(long), format='l')
@@ -74,6 +80,540 @@ def call():
     returnmvs()
     cdef object obj = returnobj()
     cdg()
-    f = Foo()
-    pf = pyfoo()
+    f = ExtClass()
+    pf = PyClass()
+
+cdef int[:] func():
+    pass
+
+cdef ExtClass get_ext_obj():
+    print 'get_ext_obj called'
+    return ExtClass.__new__(ExtClass)
+
+def test_cdef_attribute():
+    """
+    >>> test_cdef_attribute()
+    Memoryview is not initialized
+    local variable 'myview' referenced before assignment
+    get_ext_obj called
+    Memoryview is not initialized
+    <MemoryView of 'array' object>
+    """
+    cdef ExtClass extobj = ExtClass.__new__(ExtClass)
+    try:
+        print extobj.mview
+    except AttributeError, e:
+        print e.args[0]
+    else:
+        print "No AttributeError was raised"
+
+    cdef int[:] myview
+    try:
+        print myview
+    except UnboundLocalError, e:
+        print e.args[0]
+    else:
+        print "No UnboundLocalError was raised"
+
+    # uninitialized assignment is valid
+    cdef int[:] otherview = myview
+
+    try:
+        print get_ext_obj().mview
+    except AttributeError, e:
+        print e.args[0]
+    else:
+        print "No AttributeError was raised"
+
+    print ExtClass().mview
+
+
+def basic_struct(MyStruct[:] mslice):
+    """
+    See also buffmt.pyx
+
+    >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
+    1 2 3 4 5
+    >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="bbqii"))
+    1 2 3 4 5
+    """
+    buf = mslice
+    print buf[0].a, buf[0].b, buf[0].c, buf[0].d, buf[0].e
+
+def nested_struct(NestedStruct[:] mslice):
+    """
+    See also buffmt.pyx
+
+    >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
+    1 2 3 4 5
+    >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i"))
+    1 2 3 4 5
+    """
+    buf = mslice
+    print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z
+
+def packed_struct(PackedStruct[:] mslice):
+    """
+    See also buffmt.pyx
+
+    >>> packed_struct(PackedStructMockBuffer(None, [(1, 2)]))
+    1 2
+    >>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c^i}"))
+    1 2
+    >>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c=i}"))
+    1 2
+
+    """
+    buf = mslice
+    print buf[0].a, buf[0].b
+
+def nested_packed_struct(NestedPackedStruct[:] mslice):
+    """
+    See also buffmt.pyx
+
+    >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
+    1 2 3 4 5
+    >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ci^ci@i"))
+    1 2 3 4 5
+    >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i"))
+    1 2 3 4 5
+    """
+    buf = mslice
+    print buf[0].a, buf[0].b, buf[0].sub.a, buf[0].sub.b, buf[0].c
+
+
+def complex_dtype(long double complex[:] mslice):
+    """
+    >>> complex_dtype(LongComplexMockBuffer(None, [(0, -1)]))
+    -1j
+    """
+    buf = mslice
+    print buf[0]
+
+def complex_inplace(long double complex[:] mslice):
+    """
+    >>> complex_inplace(LongComplexMockBuffer(None, [(0, -1)]))
+    (1+1j)
+    """
+    buf = mslice
+    buf[0] = buf[0] + 1 + 2j
+    print buf[0]
+
+def complex_struct_dtype(LongComplex[:] mslice):
+    """
+    Note that the format string is "Zg" rather than "2g", yet a struct
+    is accessed.
+    >>> complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)]))
+    0.0 -1.0
+    """
+    buf = mslice
+    print buf[0].real, buf[0].imag
+
+def complex_struct_inplace(LongComplex[:] mslice):
+    """
+    >>> complex_struct_inplace(LongComplexMockBuffer(None, [(0, -1)]))
+    1.0 1.0
+    """
+    buf = mslice
+    buf[0].real += 1
+    buf[0].imag += 2
+    print buf[0].real, buf[0].imag
+
+#
+# Getting items and index bounds checking
+#
+def get_int_2d(int[:, :] mslice, int i, int j):
+    """
+    >>> C = IntMockBuffer("C", range(6), (2,3))
+    >>> get_int_2d(C, 1, 1)
+    acquired C
+    released C
+    4
+
+    Check negative indexing:
+    >>> get_int_2d(C, -1, 0)
+    acquired C
+    acquired C
+    released C
+    released C
+    3
+    >>> get_int_2d(C, -1, -2)
+    acquired C
+    acquired C
+    released C
+    released C
+    4
+    >>> get_int_2d(C, -2, -3)
+    acquired C
+    acquired C
+    released C
+    released C
+    0
+
+    Out-of-bounds errors:
+    >>> get_int_2d(C, 2, 0)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 0)
+    >>> get_int_2d(C, 0, -4)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 1)
+    """
+    buf = mslice
+    return buf[i, j]
+
+def set_int_2d(int[:, :] mslice, int i, int j, int value):
+    """
+    Uses get_int_2d to read back the value afterwards. For pure
+    unit test, one should support reading in MockBuffer instead.
+
+    >>> C = IntMockBuffer("C", range(6), (2,3))
+    >>> set_int_2d(C, 1, 1, 10)
+    acquired C
+    acquired C
+    released C
+    released C
+    >>> get_int_2d(C, 1, 1)
+    acquired C
+    acquired C
+    released C
+    released C
+    10
+
+    Check negative indexing:
+    >>> set_int_2d(C, -1, 0, 3)
+    acquired C
+    acquired C
+    released C
+    released C
+    >>> get_int_2d(C, -1, 0)
+    acquired C
+    acquired C
+    released C
+    released C
+    3
+
+    >>> set_int_2d(C, -1, -2, 8)
+    acquired C
+    acquired C
+    released C
+    released C
+    >>> get_int_2d(C, -1, -2)
+    acquired C
+    acquired C
+    released C
+    released C
+    8
+
+    >>> set_int_2d(C, -2, -3, 9)
+    acquired C
+    acquired C
+    released C
+    released C
+    >>> get_int_2d(C, -2, -3)
+    acquired C
+    acquired C
+    released C
+    released C
+    9
+
+    Out-of-bounds errors:
+    >>> set_int_2d(C, 2, 0, 19)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 0)
+    >>> set_int_2d(C, 0, -4, 19)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 1)
+
+    """
+    buf = mslice
+    buf[i, j] = value
+
+
+#
+# Test all kinds of indexing and flags
+#
+
+def writable(unsigned short int[:, :, :] mslice):
+    """
+    >>> R = UnsignedShortMockBuffer("R", range(27), shape=(3, 3, 3))
+    >>> writable(R)
+    acquired R
+    released R
+    >>> [str(x) for x in R.recieved_flags] # Py2/3
+    ['FORMAT', 'ND', 'STRIDES', 'WRITABLE']
+    """
+    buf = mslice
+    buf[2, 2, 1] = 23
+
+def strided(int[:] mslice):
+    """
+    >>> A = IntMockBuffer("A", range(4))
+    >>> strided(A)
+    acquired A
+    released A
+    2
+
+    Check that the suboffsets were patched back prior to release.
+    >>> A.release_ok
+    True
+    """
+    buf = mslice
+    return buf[2]
+
+def c_contig(int[::1] mslice):
+    """
+    >>> A = IntMockBuffer(None, range(4))
+    >>> c_contig(A)
+    2
+    """
+    buf = mslice
+    return buf[2]
+
+def c_contig_2d(int[:, ::1] mslice):
+    """
+    Multi-dim has seperate implementation
+
+    >>> A = IntMockBuffer(None, range(12), shape=(3,4))
+    >>> c_contig_2d(A)
+    7
+    """
+    buf = mslice
+    return buf[1, 3]
+
+def f_contig(int[::1, :] mslice):
+    """
+    >>> A = IntMockBuffer(None, range(4), shape=(2, 2), strides=(1, 2))
+    >>> f_contig(A)
+    2
+    """
+    buf = mslice
+    return buf[0, 1]
+
+def f_contig_2d(int[::1, :] mslice):
+    """
+    Must set up strides manually to ensure Fortran ordering.
+
+    >>> A = IntMockBuffer(None, range(12), shape=(4,3), strides=(1, 4))
+    >>> f_contig_2d(A)
+    7
+    """
+    buf = mslice
+    return buf[3, 1]
+
+def generic(int[::view.generic, ::view.generic] mslice1,
+            int[::view.generic, ::view.generic] mslice2):
+    """
+    >>> A = IntMockBuffer("A", [[0,1,2], [3,4,5], [6,7,8]])
+    >>> B = IntMockBuffer("B", [[0,1,2], [3,4,5], [6,7,8]], shape=(3, 3), strides=(1, 3))
+    >>> generic(A, B)
+    acquired A
+    acquired B
+    4
+    4
+    10
+    11
+    released A
+    released B
+    """
+    buf1, buf2 = mslice1, mslice2
+
+    print buf1[1, 1]
+    print buf2[1, 1]
+
+    buf1[2, -1] = 10
+    buf2[2, -1] = 11
+
+    print buf1[2, 2]
+    print buf2[2, 2]
+
+def generic_contig(int[::view.generic_contiguous, :] mslice1,
+                   int[::view.generic_contiguous, :] mslice2):
+    """
+    >>> A = IntMockBuffer("A", [[0,1,2], [3,4,5], [6,7,8]])
+    >>> B = IntMockBuffer("B", [[0,1,2], [3,4,5], [6,7,8]], shape=(3, 3), strides=(1, 3))
+    >>> generic_contig(A, B)
+    acquired A
+    acquired B
+    4
+    4
+    10
+    11
+    released A
+    released B
+    """
+    buf1, buf2 = mslice1, mslice2
+
+    print buf1[1, 1]
+    print buf2[1, 1]
+
+    buf1[2, -1] = 10
+    buf2[2, -1] = 11
+
+    print buf1[2, 2]
+    print buf2[2, 2]
+
+ctypedef int td_cy_int
+cdef extern from "bufaccess.h":
+    ctypedef td_cy_int td_h_short # Defined as short, but Cython doesn't know this!
+    ctypedef float td_h_double # Defined as double
+    ctypedef unsigned int td_h_ushort # Defined as unsigned short
+ctypedef td_h_short td_h_cy_short
+
+def printbuf_td_cy_int(td_cy_int[:] mslice, shape):
+    """
+    >>> printbuf_td_cy_int(IntMockBuffer(None, range(3)), (3,))
+    0 1 2 END
+    >>> printbuf_td_cy_int(ShortMockBuffer(None, range(3)), (3,))
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'td_cy_int' but got 'short'
+    """
+    buf = mslice
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+def printbuf_td_h_short(td_h_short[:] mslice, shape):
+    """
+    >>> printbuf_td_h_short(ShortMockBuffer(None, range(3)), (3,))
+    0 1 2 END
+    >>> printbuf_td_h_short(IntMockBuffer(None, range(3)), (3,))
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'td_h_short' but got 'int'
+    """
+    buf = mslice
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+def printbuf_td_h_cy_short(td_h_cy_short[:] mslice, shape):
+    """
+    >>> printbuf_td_h_cy_short(ShortMockBuffer(None, range(3)), (3,))
+    0 1 2 END
+    >>> printbuf_td_h_cy_short(IntMockBuffer(None, range(3)), (3,))
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'td_h_cy_short' but got 'int'
+    """
+    buf = mslice
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+def printbuf_td_h_ushort(td_h_ushort[:] mslice, shape):
+    """
+    >>> printbuf_td_h_ushort(UnsignedShortMockBuffer(None, range(3)), (3,))
+    0 1 2 END
+    >>> printbuf_td_h_ushort(ShortMockBuffer(None, range(3)), (3,))
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'td_h_ushort' but got 'short'
+    """
+    buf = mslice
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+def printbuf_td_h_double(td_h_double[:] mslice, shape):
+    """
+    >>> printbuf_td_h_double(DoubleMockBuffer(None, [0.25, 1, 3.125]), (3,))
+    0.25 1.0 3.125 END
+    >>> printbuf_td_h_double(FloatMockBuffer(None, [0.25, 1, 3.125]), (3,))
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'td_h_double' but got 'float'
+    """
+    buf = mslice
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+#
+# Object access
+#
+def addref(*args):
+    for item in args: Py_INCREF(item)
+def decref(*args):
+    for item in args: Py_DECREF(item)
+
+def get_refcount(x):
+    return (<PyObject*>x).ob_refcnt
+
+def printbuf_object(object[:] mslice, shape):
+    """
+    Only play with unique objects, interned numbers etc. will have
+    unpredictable refcounts.
+
+    ObjectMockBuffer doesn't do anything about increfing/decrefing,
+    we to the "buffer implementor" refcounting directly in the
+    testcase.
+
+    >>> a, b, c = "globally_unique_string_23234123", {4:23}, [34,3]
+    >>> get_refcount(a), get_refcount(b), get_refcount(c)
+    (2, 2, 2)
+    >>> A = ObjectMockBuffer(None, [a, b, c])
+    >>> printbuf_object(A, (3,))
+    'globally_unique_string_23234123' 2
+    {4: 23} 2
+    [34, 3] 2
+    """
+    buf = mslice
+    cdef int i
+    for i in range(shape[0]):
+        print repr(buf[i]), (<PyObject*>buf[i]).ob_refcnt
+
+def assign_to_object(object[:] mslice, int idx, obj):
+    """
+    See comments on printbuf_object above.
+
+    >>> a, b = [1, 2, 3], [4, 5, 6]
+    >>> get_refcount(a), get_refcount(b)
+    (2, 2)
+    >>> addref(a)
+    >>> A = ObjectMockBuffer(None, [1, a]) # 1, ...,otherwise it thinks nested lists...
+    >>> get_refcount(a), get_refcount(b)
+    (3, 2)
+    >>> assign_to_object(A, 1, b)
+    >>> get_refcount(a), get_refcount(b)
+    (2, 3)
+    >>> decref(b)
+    """
+    buf = mslice
+    buf[idx] = obj
+
+def assign_temporary_to_object(object[:] mslice):
+    """
+    See comments on printbuf_object above.
+
+    >>> a, b = [1, 2, 3], {4:23}
+    >>> get_refcount(a)
+    2
+    >>> addref(a)
+    >>> A = ObjectMockBuffer(None, [b, a])
+    >>> get_refcount(a)
+    3
+    >>> assign_temporary_to_object(A)
+    >>> get_refcount(a)
+    2
+
+    >>> printbuf_object(A, (2,))
+    {4: 23} 2
+    {1: 8} 2
 
+    To avoid leaking a reference in our testcase we need to
+    replace the temporary with something we can manually decref :-)
+    >>> assign_to_object(A, 1, a)
+    >>> decref(a)
+    """
+    buf = mslice
+    buf[1] = {3-2: 2+(2*4)-2}