Issue #14181: Allow memoryview construction from an object that uses the

getbuffer redirection scheme.

Lib/test/test_buffer.py

@@ -3373,6 +3373,15 @@ class TestBufferProtocol(unittest.TestCase):
         del nd
         m.release()
 
+        a = bytearray([1,2,3])
+        m = memoryview(a)
+        nd1 = ndarray(m, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        nd2 = ndarray(nd1, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        self.assertIs(nd2.obj, m)
+        self.assertRaises(BufferError, m.release)
+        del nd1, nd2
+        m.release()
+
         # chained views
         a = bytearray([1,2,3])
         m1 = memoryview(a)
@@ -3383,6 +3392,17 @@ class TestBufferProtocol(unittest.TestCase):
         del nd
         m2.release()
 
+        a = bytearray([1,2,3])
+        m1 = memoryview(a)
+        m2 = memoryview(m1)
+        nd1 = ndarray(m2, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        nd2 = ndarray(nd1, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        self.assertIs(nd2.obj, m2)
+        m1.release()
+        self.assertRaises(BufferError, m2.release)
+        del nd1, nd2
+        m2.release()
+
         # Allow changing layout while buffers are exported.
         nd = ndarray([1,2,3], shape=[3], flags=ND_VAREXPORT)
         m1 = memoryview(nd)
@@ -3418,12 +3438,82 @@ class TestBufferProtocol(unittest.TestCase):
             catch22(m1)
             self.assertEqual(m1[0], ord(b'1'))
 
+        x = ndarray(list(range(12)), shape=[2,2,3], format='l')
+        y = ndarray(x, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        z = ndarray(y, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        self.assertIs(z.obj, x)
+        with memoryview(z) as m:
+            catch22(m)
+            self.assertEqual(m[0:1].tolist(), [[[0, 1, 2], [3, 4, 5]]])
+
+        # Test garbage collection.
+        for flags in (0, ND_REDIRECT):
+            x = bytearray(b'123')
+            with memoryview(x) as m1:
+                del x
+                y = ndarray(m1, getbuf=PyBUF_FULL_RO, flags=flags)
+                with memoryview(y) as m2:
+                    del y
+                    z = ndarray(m2, getbuf=PyBUF_FULL_RO, flags=flags)
+                    with memoryview(z) as m3:
+                        del z
+                        catch22(m3)
+                        catch22(m2)
+                        catch22(m1)
+                        self.assertEqual(m1[0], ord(b'1'))
+                        self.assertEqual(m2[1], ord(b'2'))
+                        self.assertEqual(m3[2], ord(b'3'))
+                        del m3
+                    del m2
+                del m1
+
+            x = bytearray(b'123')
+            with memoryview(x) as m1:
+                del x
+                y = ndarray(m1, getbuf=PyBUF_FULL_RO, flags=flags)
+                with memoryview(y) as m2:
+                    del y
+                    z = ndarray(m2, getbuf=PyBUF_FULL_RO, flags=flags)
+                    with memoryview(z) as m3:
+                        del z
+                        catch22(m1)
+                        catch22(m2)
+                        catch22(m3)
+                        self.assertEqual(m1[0], ord(b'1'))
+                        self.assertEqual(m2[1], ord(b'2'))
+                        self.assertEqual(m3[2], ord(b'3'))
+                        del m1, m2, m3
+
         # XXX If m1 has exports, raise BufferError.
         # x = bytearray(b'123')
         # with memoryview(x) as m1:
         #     ex = ndarray(m1)
         #     m1[0] == ord(b'1')
 
+    def test_memoryview_redirect(self):
+
+        nd = ndarray([1.0 * x for x in range(12)], shape=[12], format='d')
+        a = array.array('d', [1.0 * x for x in range(12)])
+
+        for x in (nd, a):
+            y = ndarray(x, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+            z = ndarray(y, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+            m = memoryview(z)
+
+            self.assertIs(y.obj, x)
+            self.assertIs(z.obj, x)
+            self.assertIs(m.obj, x)
+
+            self.assertEqual(m, x)
+            self.assertEqual(m, y)
+            self.assertEqual(m, z)
+
+            self.assertEqual(m[1:3], x[1:3])
+            self.assertEqual(m[1:3], y[1:3])
+            self.assertEqual(m[1:3], z[1:3])
+            del y, z
+            self.assertEqual(m[1:3], x[1:3])
+
     def test_issue_7385(self):
         x = ndarray([1,2,3], shape=[3], flags=ND_GETBUF_FAIL)
         self.assertRaises(BufferError, memoryview, x)

Modules/_testbuffer.c

@@ -53,14 +53,14 @@ static PyTypeObject NDArray_Type;
 #define ND_SCALAR       0x008   /* scalar: ndim = 0 */
 #define ND_PIL          0x010   /* convert to PIL-style array (suboffsets) */
 #define ND_GETBUF_FAIL  0x020   /* test issue 7385 */
+#define ND_REDIRECT     0x040   /* redirect buffer requests */
 
 /* Default: NumPy style (strides), read-only, no var-export, C-style layout */
 #define ND_DEFAULT      0x0
 
 /* Internal flags for the base buffer */
-#define ND_C            0x040   /* C contiguous layout (default) */
-#define ND_OWN_ARRAYS   0x080   /* consumer owns arrays */
-#define ND_UNUSED       0x100   /* initializer */
+#define ND_C            0x080   /* C contiguous layout (default) */
+#define ND_OWN_ARRAYS   0x100   /* consumer owns arrays */
 
 /* ndarray properties */
 #define ND_IS_CONSUMER(nd) \
@@ -1290,7 +1290,7 @@ ndarray_init(PyObject *self, PyObject *args, PyObject *kwds)
     PyObject *strides = NULL; /* number of bytes to the next elt in each dim */
     Py_ssize_t offset = 0;            /* buffer offset */
     PyObject *format = simple_format; /* struct module specifier: "B" */
-    int flags = ND_UNUSED;            /* base buffer and ndarray flags */
+    int flags = ND_DEFAULT;           /* base buffer and ndarray flags */
 
     int getbuf = PyBUF_UNUSED; /* re-exporter: getbuffer request flags */
 
@@ -1302,10 +1302,10 @@ ndarray_init(PyObject *self, PyObject *args, PyObject *kwds)
     /* NDArrayObject is re-exporter */
     if (PyObject_CheckBuffer(v) && shape == NULL) {
         if (strides || offset || format != simple_format ||
-            flags != ND_UNUSED) {
+            !(flags == ND_DEFAULT || flags == ND_REDIRECT)) {
             PyErr_SetString(PyExc_TypeError,
-               "construction from exporter object only takes a single "
-               "additional getbuf argument");
+               "construction from exporter object only takes 'obj', 'getbuf' "
+               "and 'flags' arguments");
             return -1;
         }
 
@@ -1315,6 +1315,7 @@ ndarray_init(PyObject *self, PyObject *args, PyObject *kwds)
             return -1;
 
         init_flags(nd->head);
+        nd->head->flags |= flags;
 
         return 0;
     }
@@ -1333,8 +1334,6 @@ ndarray_init(PyObject *self, PyObject *args, PyObject *kwds)
         return -1;
     }
 
-    if (flags == ND_UNUSED)
-        flags = ND_DEFAULT;
     if (flags & ND_VAREXPORT) {
         nd->flags |= ND_VAREXPORT;
         flags &= ~ND_VAREXPORT;
@@ -1357,7 +1356,7 @@ ndarray_push(PyObject *self, PyObject *args, PyObject *kwds)
     PyObject *strides = NULL; /* number of bytes to the next elt in each dim */
     PyObject *format = simple_format;  /* struct module specifier: "B" */
     Py_ssize_t offset = 0;             /* buffer offset */
-    int flags = ND_UNUSED;             /* base buffer flags */
+    int flags = ND_DEFAULT;            /* base buffer flags */
 
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|OnOi", kwlist,
             &items, &shape, &strides, &offset, &format, &flags))
@@ -1423,6 +1422,11 @@ ndarray_getbuf(NDArrayObject *self, Py_buffer *view, int flags)
     Py_buffer *base = &ndbuf->base;
     int baseflags = ndbuf->flags;
 
+    /* redirect mode */
+    if (base->obj != NULL && (baseflags&ND_REDIRECT)) {
+        return PyObject_GetBuffer(base->obj, view, flags);
+    }
+
     /* start with complete information */
     *view = *base;
     view->obj = NULL;
@@ -2654,6 +2658,7 @@ PyInit__testbuffer(void)
     PyModule_AddIntConstant(m, "ND_SCALAR", ND_SCALAR);
     PyModule_AddIntConstant(m, "ND_PIL", ND_PIL);
     PyModule_AddIntConstant(m, "ND_GETBUF_FAIL", ND_GETBUF_FAIL);
+    PyModule_AddIntConstant(m, "ND_REDIRECT", ND_REDIRECT);
 
     PyModule_AddIntConstant(m, "PyBUF_SIMPLE", PyBUF_SIMPLE);
     PyModule_AddIntConstant(m, "PyBUF_WRITABLE", PyBUF_WRITABLE);

Objects/memoryobject.c

@@ -91,9 +91,6 @@ _PyManagedBuffer_FromObject(PyObject *base)
         return NULL;
     }
 
-    /* Assume that master.obj is a new reference to base. */
-    assert(mbuf->master.obj == base);
-
     return (PyObject *)mbuf;
 }