Fixed and cleaned buffer acquisition (but should do more); well on the way for indirect access

Cython/Compiler/Buffer.py

@@ -7,7 +7,7 @@ from Cython.Utils import EncodedString
 from Cython.Compiler.Errors import CompileError
 import PyrexTypes
 from sets import Set as set
-
+from textwrap import dedent
 
 class IntroduceBufferAuxiliaryVars(CythonTransform):
 
@@ -19,12 +19,14 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
 
     def __call__(self, node):
         assert isinstance(node, ModuleNode)
+        self.max_ndim = 0
         result = super(IntroduceBufferAuxiliaryVars, self).__call__(node)
         if self.buffers_exists:
             if "endian.h" not in node.scope.include_files:
                 node.scope.include_files.append("endian.h")
             use_py2_buffer_functions(node.scope)
-            node.scope.use_utility_code(buffer_boundsfail_error_utility_code)
+            use_empty_bufstruct_code(node.scope, self.max_ndim)
+            node.scope.use_utility_code(access_utility_code)
         return result
 
 
@@ -48,6 +50,8 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
         for entry in bufvars:
             name = entry.name
             buftype = entry.type
+            if buftype.ndim > self.max_ndim:
+                self.max_ndim = buftype.ndim
 
             # Get or make a type string checker
             tschecker = buffer_type_checker(buftype.dtype, scope)
@@ -59,19 +63,23 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
 
             bufinfo.used = True
 
-            def var(prefix, idx):
+            def var(prefix, idx, initval):
                 cname = scope.mangle(prefix, "%d_%s" % (idx, name))
                 result = scope.declare_var("$%s" % cname, PyrexTypes.c_py_ssize_t_type,
                                          node.pos, cname=cname, is_cdef=True)
 
-                result.init = "0"
+                result.init = initval
                 if entry.is_arg:
                     result.used = True
                 return result
             
-            stridevars = [var(Naming.bufstride_prefix, i) for i in range(entry.type.ndim)]
-            shapevars = [var(Naming.bufshape_prefix, i) for i in range(entry.type.ndim)]            
+            stridevars = [var(Naming.bufstride_prefix, i, "0") for i in range(entry.type.ndim)]
+            shapevars = [var(Naming.bufshape_prefix, i, "0") for i in range(entry.type.ndim)]            
+            suboffsetvars = [var(Naming.bufsuboffset_prefix, i, "-1") for i in range(entry.type.ndim)]
             entry.buffer_aux = Symtab.BufferAux(bufinfo, stridevars, shapevars, tschecker)
+            entry.buffer_aux.lookup = get_buf_lookup_full(scope, entry.type.ndim)
+            entry.buffer_aux.suboffsetvars = suboffsetvars
+            entry.buffer_aux.get_buffer_cname = tschecker
             
         scope.buffer_entries = bufvars
         self.scope = scope
@@ -99,31 +107,24 @@ def used_buffer_aux_vars(entry):
     buffer_aux.buffer_info_var.used = True
     for s in buffer_aux.shapevars: s.used = True
     for s in buffer_aux.stridevars: s.used = True
+    for s in buffer_aux.suboffsetvars: s.used = True
 
 def put_unpack_buffer_aux_into_scope(buffer_aux, code):
     bufstruct = buffer_aux.buffer_info_var.cname
 
-    code.putln(" ".join(["%s = %s.strides[%d];" %
-                         (s.cname, bufstruct, idx)
-                         for idx, s in enumerate(buffer_aux.stridevars)]))
-    code.putln(" ".join(["%s = %s.shape[%d];" %
-                         (s.cname, bufstruct, idx)
-                         for idx, s in enumerate(buffer_aux.shapevars)]))
-
-def put_zero_buffer_aux_into_scope(buffer_aux, code):
-    # If new buffer is None, set up to access 0
-    # for a "safer segfault" on access
-    code.putln("%s.buf = 0;" % buffer_aux.buffer_info_var.cname) 
-    code.putln(" ".join(["%s = 0;" % s.cname
-                         for s in buffer_aux.stridevars]))
-    code.putln(" ".join(["%s = 0;" % s.cname
-                         for s in buffer_aux.shapevars]))    
+    # __pyx_bstride_0_buf = __pyx_bstruct_buf.strides[0] and so on
+
+    for field, vars in (("strides", buffer_aux.stridevars),
+                        ("shape", buffer_aux.shapevars),
+                        ("suboffsets", buffer_aux.suboffsetvars)):
+        code.putln(" ".join(["%s = %s.%s[%d];" %
+                             (s.cname, bufstruct, field, idx)
+                             for idx, s in enumerate(vars)]))
 
 def getbuffer_cond_code(obj_cname, buffer_aux, flags, ndim):
     bufstruct = buffer_aux.buffer_info_var.cname
-    checker = buffer_aux.tschecker
-    return "PyObject_GetBuffer(%s, &%s, %s) == -1  || %s(&%s, %d) == -1" % (
-        obj_cname, bufstruct, flags, checker, bufstruct, ndim)
+    return "%s(%s, &%s, %s, %d) == -1" % (
+        buffer_aux.get_buffer_cname, obj_cname, bufstruct, flags, ndim)
                    
 def put_acquire_arg_buffer(entry, code, pos):
     buffer_aux = entry.buffer_aux
@@ -131,10 +132,7 @@ def put_acquire_arg_buffer(entry, code, pos):
     bufstruct = buffer_aux.buffer_info_var.cname
     flags = get_flags(buffer_aux, entry.type)
     # Acquire any new buffer
-    code.put('if (%s != Py_None) ' % cname)
-    code.begin_block()
-    code.putln('%s.buf = 0;' % bufstruct) # PEP requirement
-    code.put(code.error_goto_if(getbuffer_cond_code(cname,
+    code.putln(code.error_goto_if(getbuffer_cond_code(cname,
                                                     buffer_aux,
                                                     flags,
                                                     entry.type.ndim),
@@ -142,17 +140,36 @@ def put_acquire_arg_buffer(entry, code, pos):
     # An exception raised in arg parsing cannot be catched, so no
     # need to do care about the buffer then.
     put_unpack_buffer_aux_into_scope(buffer_aux, code)
-    code.end_block()
 
 def put_release_buffer(entry, code):
     code.putln("if (%s != Py_None) PyObject_ReleaseBuffer(%s, &%s);" % (
         entry.cname, entry.cname, entry.buffer_aux.buffer_info_var.cname))
 
-def put_assign_to_buffer(lhs_cname, rhs_cname, buffer_aux, buffer_type,
+def put_assign_to_buffer(lhs_cname, rhs_cname, retcode_cname, buffer_aux, buffer_type,
                          is_initialized, pos, code):
+    """
+    Generate code for reassigning a buffer variables. This only deals with getting
+    the buffer auxiliary structure and variables set up correctly, the assignment
+    itself and refcounting is the responsibility of the caller.
+
+    However, the assignment operation may throw an exception so that the reassignment
+    never happens.
+    
+    Depending on the circumstances there are two possible outcomes:
+    - Old buffer released, new acquired, rhs assigned to lhs
+    - Old buffer released, new acquired which fails, reaqcuire old lhs buffer
+      (which may or may not succeed).
+    """
+    
     bufstruct = buffer_aux.buffer_info_var.cname
     flags = get_flags(buffer_aux, buffer_type)
 
+    getbuffer = "%s(%%s, &%s, %s, %d)" % (buffer_aux.get_buffer_cname,
+                                          # note: object is filled in later
+                                          bufstruct,
+                                          flags,
+                                          buffer_type.ndim)
+
     if is_initialized:
         # Release any existing buffer
         code.put('if (%s != Py_None) ' % lhs_cname)
@@ -160,43 +177,50 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buffer_aux, buffer_type,
         code.putln('PyObject_ReleaseBuffer(%s, &%s);' % (
             lhs_cname, bufstruct))
         code.end_block()
-    # Acquire any new buffer
-    code.put('if (%s != Py_None) ' % rhs_cname)
-    code.begin_block()
-    code.putln('%s.buf = 0;' % bufstruct) # PEP requirement
-    code.put('if (%s) ' % code.unlikely(getbuffer_cond_code(rhs_cname, buffer_aux, flags, buffer_type.ndim)))
-    code.begin_block()
+        # Acquire
+        code.putln("%s = %s;" % (retcode_cname, getbuffer % rhs_cname))
         # If acquisition failed, attempt to reacquire the old buffer
         # before raising the exception. A failure of reacquisition
         # will cause the reacquisition exception to be reported, one
         # can consider working around this later.
-    if is_initialized:
-        put_zero_buffer_aux_into_scope(buffer_aux, code)
-        code.put('if (%s != Py_None && (%s)) ' % (rhs_cname, 
-            getbuffer_cond_code(rhs_cname, buffer_aux, flags, buffer_type.ndim)))
+        code.putln('if (%s) ' % (code.unlikely("%s < 0" % retcode_cname)))
+        code.begin_block()
+        # In anticipation of a better temp system, create non-consistent C code for now
+        code.putln('PyObject *__pyx_type, *__pyx_value, *__pyx_tb;')
+        code.putln('PyErr_Fetch(&__pyx_type, &__pyx_value, &__pyx_tb);')
+        code.put('if (%s) ' % code.unlikely("%s == -1" % (getbuffer % lhs_cname)))
         code.begin_block()
-        put_zero_buffer_aux_into_scope(buffer_aux, code)
+        code.putln('Py_XDECREF(__pyx_type); Py_XDECREF(__pyx_value); Py_XDECREF(__pyx_tb);')
+        code.putln('PyErr_Format(PyExc_ValueError, "Buffer acquisition failed on assignment; and then reacquiring the old buffer failed too!");')
+        code.putln('} else {')
+        code.putln('PyErr_Restore(__pyx_type, __pyx_value, __pyx_tb);')
+        code.end_block()
+        # Unpack indices
         code.end_block()
+        put_unpack_buffer_aux_into_scope(buffer_aux, code)
+        code.putln(code.error_goto_if_neg(retcode_cname, pos))
     else:
-        # our entry had no previous value, so set to None when acquisition fails
-        code.putln('%s = Py_None; Py_INCREF(Py_None);' % lhs_cname)
+        # Our entry had no previous value, so set to None when acquisition fails.
+        # In this case, auxiliary vars should be set up right in initialization to a zero-buffer,
+        # so it suffices to set the buf field to NULL.
+        code.putln('if (%s) {' % code.unlikely("%s == -1" % (getbuffer % rhs_cname)))
+        code.putln('%s = Py_None; Py_INCREF(Py_None); %s.buf = NULL;' % (lhs_cname, bufstruct))
         code.putln(code.error_goto(pos))
-    code.end_block() # acquisition failure
+        code.put('} else {')
         # Unpack indices
         put_unpack_buffer_aux_into_scope(buffer_aux, code)
-    code.putln('} else {')
-    # If new buffer is None, set up to access 0
-    # for a "safer segfault" on access
-    put_zero_buffer_aux_into_scope(buffer_aux, code)
-    code.end_block()
-
-    # Everything is ok, assign object variable
-    code.putln("%s = %s;" % (lhs_cname, rhs_cname))
+        code.putln('}')
 
 
 def put_access(entry, index_types, index_cnames, tmp_cname, pos, code):
     """Returns a c string which can be used to access the buffer
-    for reading or writing"""
+    for reading or writing.
+
+    As the bounds checking can have any number of combinations of unsigned
+    arguments, smart optimizations etc. we insert it directly in the function
+    body. The lookup however is delegated to a inline function that is instantiated
+    once per ndim (lookup with suboffsets tend to get quite complicated).
+    """
     bufaux = entry.buffer_aux
     bufstruct = bufaux.buffer_info_var.cname
     # Check bounds and fix negative indices
@@ -237,13 +261,24 @@ def put_access(entry, index_types, index_cnames, tmp_cname, pos, code):
                          for idx, stride in
                          zip(index_cnames, bufaux.stridevars)])
     ptrcode = "(%s.buf + %s)" % (bufstruct, offset)
+    ptrcode = "%s(%s.buf, %s)" % (bufaux.lookup, bufstruct, 
+                          ", ".join([", ".join([i, s.cname, o.cname]) for i, s, o in
+                                     zip(index_cnames, bufaux.stridevars, bufaux.suboffsetvars)]))
     valuecode = "*%s" % entry.type.buffer_ptr_type.cast_code(ptrcode)
     return valuecode
 
 
+
+def use_empty_bufstruct_code(env, max_ndim):
+    code = dedent("""
+        Py_ssize_t __Pyx_zeros[] = {%s};
+        Py_ssize_t __Pyx_minusones[] = {%s};
+    """) % (", ".join(["0"] * max_ndim), ", ".join(["-1"] * max_ndim))
+    env.use_utility_code([code, ""])
+
 # Utility function to set the right exception
 # The caller should immediately goto_error
-buffer_boundsfail_error_utility_code = [
+access_utility_code = [
 """\
 static void __Pyx_BufferIndexError(int axis); /*proto*/
 ""","""\
@@ -253,7 +288,6 @@ static void __Pyx_BufferIndexError(int axis) {
 }
 """]
 
-
 #
 # Buffer type checking. Utility code for checking that acquired
 # buffers match our assumptions. We only need to check ndim and
@@ -261,10 +295,18 @@ static void __Pyx_BufferIndexError(int axis) {
 # exporter.
 #
 buffer_check_utility_code = ["""\
+static void __Pyx_ZeroBuffer(Py_buffer* buf); /*proto*/
 static const char* __Pyx_ConsumeWhitespace(const char* ts); /*proto*/
 static const char* __Pyx_BufferTypestringCheckEndian(const char* ts); /*proto*/
 static void __Pyx_BufferNdimError(Py_buffer* buffer, int expected_ndim); /*proto*/
 """, """
+static void __Pyx_ZeroBuffer(Py_buffer* buf) {
+  buf->buf = NULL;
+  buf->strides = __Pyx_zeros;
+  buf->shape = __Pyx_zeros;
+  buf->suboffsets = __Pyx_minusones;
+}
+
 static const char* __Pyx_ConsumeWhitespace(const char* ts) {
   while (1) {
     switch (*ts) {
@@ -310,6 +352,36 @@ static void __Pyx_BufferNdimError(Py_buffer* buffer, int expected_ndim) {
 """]
 
 
+
+def get_buf_lookup_full(env, nd):
+    """
+    Generates and registers as utility a buffer lookup function for the right number
+    of dimensions. The function gives back a void* at the right location.
+    """
+    name = "__Pyx_BufPtrFull_%dd" % nd
+    if not env.has_utility_code(name):
+        # _i_ndex, _s_tride, sub_o_ffset
+        args = ", ".join(["Py_ssize_t i%d, Py_ssize_t s%d, Py_ssize_t o%d" % (i, i, i) for i in range(nd)])
+        proto = dedent("""\
+        static INLINE void* %s(void* buf, %s);
+        """) % (name, args) 
+
+        func = dedent("""
+        static INLINE void* %s(void* buf, %s) {
+          char* ptr = (char*)buf;
+        """) % (name, args) + "".join([dedent("""\
+          ptr += s%d * i%d;
+          if (o%d >= 0) ptr = *((char**)ptr) + o%d; 
+        """) % (i, i, i, i) for i in range(nd)]
+        ) + "\nreturn ptr;\n}"
+
+        env.use_utility_code([proto, func], name=name)
+        
+    return name
+
+
+
+
 #
 # Utils for creating type string checkers
 #
@@ -372,35 +444,54 @@ static const char* %s(const char* ts) {
 
     return name
 
-def get_ts_check_simple(dtype, env):
-    # Check whole string for single unnamed item
-    name = "__Pyx_BufferTypestringCheck_simple_%s" % mangle_dtype_name(dtype)
+def get_getbuffer_code(dtype, env):
+    """
+    Generate a utility function for getting a buffer for the given dtype.
+    The function will:
+    - Call PyObject_GetBuffer
+    - Check that ndim matched the expected value
+    - Check that the format string is right
+    - Set suboffsets to all -1 if it is returned as NULL.
+    """
+
+    name = "__Pyx_GetBuffer_%s" % mangle_dtype_name(dtype)
     if not env.has_utility_code(name):
+        env.use_utility_code(buffer_check_utility_code)
         itemchecker = get_ts_check_item(dtype, env)
-        utilcode = ["""
-static int %s(Py_buffer* buf, int e_nd); /*proto*/
-""" % name,"""
-static int %(name)s(Py_buffer* buf, int e_nd) {
-  const char* ts = buf->format;
-  if (buf->ndim != e_nd) {
-    __Pyx_BufferNdimError(buf, e_nd);
-    return -1;
+        utilcode = [dedent("""
+        static int %s(PyObject* obj, Py_buffer* buf, int flags, int nd); /*proto*/
+        """) % name, dedent("""
+        static int %(name)s(PyObject* obj, Py_buffer* buf, int flags, int nd) {
+          const char* ts;
+          if (obj == Py_None) {
+            __Pyx_ZeroBuffer(buf);
+            return 0;
+          }
+          buf->buf = NULL;
+          if (PyObject_GetBuffer(obj, buf, flags) == -1) goto fail;
+          if (buf->ndim != nd) {
+            __Pyx_BufferNdimError(buf, nd);
+            goto fail;
           }
+          ts = buf->format;
           ts = __Pyx_ConsumeWhitespace(ts);
           ts = __Pyx_BufferTypestringCheckEndian(ts);
-  if (!ts) return -1;
+          if (!ts) goto fail;
           ts = __Pyx_ConsumeWhitespace(ts);
           ts = %(itemchecker)s(ts);
-  if (!ts) return -1;
+          if (!ts) goto fail;
           ts = __Pyx_ConsumeWhitespace(ts);
           if (*ts != 0) {
             PyErr_Format(PyExc_ValueError,
               "Expected non-struct buffer data type (rejecting on '%%s')", ts);
-    return -1;
+            goto fail;
           }
+          if (buf->suboffsets == NULL) buf->suboffsets = __Pyx_minusones;
           return 0;
-}""" % locals()]
-        env.use_utility_code(buffer_check_utility_code)
+        fail:;
+          __Pyx_ZeroBuffer(buf);
+          return -1;
+        }""") % locals()]
         env.use_utility_code(utilcode, name)
     return name
 
@@ -410,7 +501,7 @@ def buffer_type_checker(dtype, env):
         assert False
     elif dtype.is_int or dtype.is_float:
         # This includes simple typedef-ed types
-        funcname = get_ts_check_simple(dtype, env)
+        funcname = get_getbuffer_code(dtype, env)
     else:
         assert False
     return funcname

Cython/Compiler/ExprNodes.py

@@ -890,7 +890,8 @@ class NameNode(AtomicExprNode):
             # safe than sorry. Feel free to change this.
             import Buffer
             self.new_buffer_temp = Symtab.new_temp(self.entry.type)
-            self.temps = [self.new_buffer_temp]
+            self.retcode_temp = Symtab.new_temp(PyrexTypes.c_int_type)
+            self.temps = [self.new_buffer_temp, self.retcode_temp]
             Buffer.used_buffer_aux_vars(self.entry)
                 
     def analyse_rvalue_entry(self, env):
@@ -1035,6 +1036,16 @@ class NameNode(AtomicExprNode):
                 rhs.generate_disposal_code(code)
                 
         else:
+            if self.type.is_buffer:
+                # Generate code for doing the buffer release/acquisition.
+                # This might raise an exception in which case the assignment (done
+                # below) will not happen.
+                #
+                # The reason this is not in a typetest-like node is because the
+                # variables that the acquired buffer info is stored to is allocated
+                # per entry and coupled with it.
+                self.generate_acquire_buffer(rhs, code)
+
             if self.type.is_pyobject:
                 rhs.make_owned_reference(code)
                 #print "NameNode.generate_assignment_code: to", self.name ###
@@ -1050,9 +1061,6 @@ class NameNode(AtomicExprNode):
                             code.put_xdecref(self.result_code, self.ctype())
                     else:
                         code.put_decref(self.result_code, self.ctype())
-            if self.type.is_buffer:
-                self.generate_acquire_buffer(rhs, code)
-            else:
             code.putln('%s = %s;' % (self.result_code, rhs.result_as(self.ctype())))
             if debug_disposal_code:
                 print("NameNode.generate_assignment_code:")
@@ -1066,7 +1074,7 @@ class NameNode(AtomicExprNode):
         code.putln('%s = %s;' % (rhstmp, rhs.result_as(self.ctype())))
 
         import Buffer
-        Buffer.put_assign_to_buffer(self.result_code, rhstmp, buffer_aux, self.entry.type,
+        Buffer.put_assign_to_buffer(self.result_code, rhstmp, self.retcode_temp.cname, buffer_aux, self.entry.type,
                                     is_initialized=not self.skip_assignment_decref,
                                     pos=self.pos, code=code)
         code.putln("%s = 0;" % rhstmp)

Cython/Compiler/Naming.py

@@ -34,7 +34,7 @@ var_prefix        = pyrex_prefix + "v_"
 bufstruct_prefix  = pyrex_prefix + "bstruct_"
 bufstride_prefix  = pyrex_prefix + "bstride_"
 bufshape_prefix   = pyrex_prefix + "bshape_"
-bufoffset_prefix  = pyrex_prefix + "boffset_"
+bufsuboffset_prefix  = pyrex_prefix + "boffset_"
 vtable_prefix     = pyrex_prefix + "vtable_"
 vtabptr_prefix    = pyrex_prefix + "vtabptr_"
 vtabstruct_prefix = pyrex_prefix + "vtabstruct_"

tests/run/bufaccess.pyx

@@ -26,6 +26,9 @@ def testcase(func):
     __test__[func.__name__] = setup_string + func.__doc__
     return func
 
+def testcas(a):
+    pass
+
 @testcase
 def acquire_release(o1, o2):
     """
@@ -34,11 +37,16 @@ def acquire_release(o1, o2):
     released A
     acquired B
     released B
+    >>> acquire_release(None, None)
+    >>> acquire_release(None, B)
+    acquired B
+    released B
     """
     cdef object[int] buf
     buf = o1
     buf = o2
 
+#TODO!
 #@testcase
 def acquire_raise(o):
     """
@@ -60,6 +68,134 @@ def acquire_raise(o):
     o.printlog()
     raise Exception("on purpose")
 
+@testcase
+def acquire_failure1():
+    """
+    >>> acquire_failure1()
+    acquired working
+    0 3
+    0 3
+    released working
+    """
+    cdef object[int] buf
+    buf = IntMockBuffer("working", range(4))
+    print buf[0], buf[3]
+    try:
+        buf = ErrorBuffer()
+        assert False
+    except Exception:
+        print buf[0], buf[3]
+
+@testcase
+def acquire_failure2():
+    """
+    >>> acquire_failure2()
+    acquired working
+    0 3
+    0 3
+    released working
+    """
+    cdef object[int] buf = IntMockBuffer("working", range(4))
+    print buf[0], buf[3]
+    try:
+        buf = ErrorBuffer()
+        assert False
+    except Exception:
+        print buf[0], buf[3]
+
+@testcase
+def acquire_failure3():
+    """
+    >>> acquire_failure3()
+    acquired working
+    0 3
+    released working
+    acquired working
+    0 3
+    released working
+    """
+    cdef object[int] buf
+    buf = IntMockBuffer("working", range(4))
+    print buf[0], buf[3]
+    try:
+        buf = 3
+        assert False
+    except Exception:
+        print buf[0], buf[3]
+
+@testcase
+def acquire_failure4():
+    """
+    >>> acquire_failure4()
+    acquired working
+    0 3
+    released working
+    acquired working
+    0 3
+    released working
+    """
+    cdef object[int] buf = IntMockBuffer("working", range(4))
+    print buf[0], buf[3]
+    try:
+        buf = 2
+        assert False
+    except Exception:
+        print buf[0], buf[3]
+
+@testcase
+def acquire_failure5():
+    """
+    >>> acquire_failure5()
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer acquisition failed on assignment; and then reacquiring the old buffer failed too!
+    """
+    cdef object[int] buf
+    buf = IntMockBuffer("working", range(4))
+    buf.fail = True
+    buf = 3
+
+
+@testcase
+def acquire_nonbuffer1(first, second=None):
+    """
+    >>> acquire_nonbuffer1(3)
+    Traceback (most recent call last):
+      ...
+    TypeError: 'int' does not have the buffer interface
+    >>> acquire_nonbuffer1(type)
+    Traceback (most recent call last):
+      ...
+    TypeError: 'type' does not have the buffer interface
+    >>> acquire_nonbuffer1(None, 2)
+    Traceback (most recent call last):
+      ...
+    TypeError: 'int' does not have the buffer interface
+    """
+    cdef object[int] buf
+    buf = first
+    buf = second
+
+@testcase
+def acquire_nonbuffer2():
+    """
+    >>> acquire_nonbuffer2()
+    acquired working
+    0 3
+    released working
+    acquired working
+    0 3
+    released working
+    """
+    cdef object[int] buf = IntMockBuffer("working", range(4))
+    print buf[0], buf[3]
+    try:
+        buf = ErrorBuffer
+        assert False
+    except Exception:
+        print buf[0], buf[3]
+
+
 @testcase
 def as_argument(object[int] bufarg, int n):
     """
@@ -331,14 +467,19 @@ available_flags = (
     ('WRITABLE', python_buffer.PyBUF_WRITABLE)
 )
 
+cimport stdio
+
 cdef class MockBuffer:
     cdef object format
-    cdef char* buffer
+    cdef void* buffer
     cdef int len, itemsize, ndim
     cdef Py_ssize_t* strides
     cdef Py_ssize_t* shape
+    cdef Py_ssize_t* suboffsets
     cdef object label, log
+    
     cdef readonly object recieved_flags
+    cdef public object fail
     
     def __init__(self, label, data, shape=None, strides=None, format=None):
         self.label = label
@@ -356,27 +497,78 @@ cdef class MockBuffer:
                 cumprod *= s
             strides.reverse()
         strides = [x * self.itemsize for x in strides]
+        suboffsets = [-1] * len(shape)
+
+        datashape = [len(data)]
+        p = data
+        while True:
+            p = p[0]
+            if isinstance(p, list): datashape.append(len(p))
+            else: break
+        if len(datashape) > 1:
+            # indirect access
+            self.ndim = len(datashape)
+            shape = datashape
+            self.buffer = self.create_indirect_buffer(data, shape)
+            self.suboffsets = self.list_to_sizebuf(suboffsets)
+        else:
+            # strided and/or simple access
+            self.buffer = self.create_buffer(data)
+            self.ndim = len(shape)
+            self.suboffsets = NULL
+            
         self.format = format
         self.len = len(data) * self.itemsize
-        self.buffer = <char*>stdlib.malloc(self.len)
-        self.fill_buffer(data)
-        self.ndim = len(shape)
-        self.strides = <Py_ssize_t*>stdlib.malloc(self.ndim * sizeof(Py_ssize_t))
-        for i, x in enumerate(strides):
-            self.strides[i] = x
-        self.shape = <Py_ssize_t*>stdlib.malloc(self.ndim * sizeof(Py_ssize_t))
-        for i, x in enumerate(shape):
-            self.shape[i] = x
+
+        self.strides = self.list_to_sizebuf(strides)
+        self.shape = self.list_to_sizebuf(shape)
+            
     def __dealloc__(self):
         stdlib.free(self.strides)
         stdlib.free(self.shape)
+        if self.suboffsets != NULL:
+            stdlib.free(self.suboffsets)
+            # must recursively free indirect...
+        else:
+            stdlib.free(self.buffer)
+    
+    cdef void* create_buffer(self, data):
+        cdef char* buf = <char*>stdlib.malloc(len(data) * self.itemsize)
+        cdef char* it = buf
+        for value in data:
+            self.write(it, value)
+            it += self.itemsize
+        return buf
+
+    cdef void* create_indirect_buffer(self, data, shape):
+        cdef void** buf
+
+        assert shape[0] == len(data)
+        if len(shape) == 1:
+            return self.create_buffer(data)
+        else:
+            shape = shape[1:]
+            buf = <void**>stdlib.malloc(len(data) * sizeof(void*))
+            for idx, subdata in enumerate(data):
+                buf[idx] = self.create_indirect_buffer(subdata, shape)
+            return buf
+
+    cdef Py_ssize_t* list_to_sizebuf(self, l):
+        cdef Py_ssize_t* buf = <Py_ssize_t*>stdlib.malloc(len(l) * sizeof(Py_ssize_t))
+        for i, x in enumerate(l):
+            buf[i] = x
+        return buf
 
     def __getbuffer__(MockBuffer self, Py_buffer* buffer, int flags):
+        if self.fail:
+            raise ValueError("Failing on purpose")
+        
         if buffer is NULL:
             print u"locking!"
             return
 
         self.recieved_flags = []
+        cdef int value
         for name, value in available_flags:
             if (value & flags) == value:
                 self.recieved_flags.append(name)
@@ -388,7 +580,7 @@ cdef class MockBuffer:
         buffer.ndim = self.ndim
         buffer.shape = self.shape
         buffer.strides = self.strides
-        buffer.suboffsets = NULL
+        buffer.suboffsets = self.suboffsets
         buffer.itemsize = self.itemsize
         buffer.internal = NULL
         msg = "acquired %s" % self.label
@@ -400,12 +592,6 @@ cdef class MockBuffer:
         print msg
         self.log += msg + "\n"
 
-    cdef fill_buffer(self, object data):
-        cdef char* it = self.buffer
-        for value in data:
-            self.write(it, value)
-            it += self.itemsize
-
     def printlog(self):
         print self.log,