Issue #12834: Fix PyBuffer_ToContiguous() for non-contiguous arrays.

Include/abstract.h

@@ -535,11 +535,12 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
 
 
 
+     /* Implementation in memoryobject.c */
      PyAPI_FUNC(int) PyBuffer_ToContiguous(void *buf, Py_buffer *view,
-                                           Py_ssize_t len, char fort);
+                                           Py_ssize_t len, char order);
 
      PyAPI_FUNC(int) PyBuffer_FromContiguous(Py_buffer *view, void *buf,
-                                             Py_ssize_t len, char fort);
+                                             Py_ssize_t len, char order);
 
 
     /* Copy len bytes of data from the contiguous chunk of memory

Misc/NEWS

@@ -10,6 +10,8 @@ What's New in Python 3.3.0 Beta 2?
 Core and Builtins
 -----------------
 
+- Issue #12834: Fix PyBuffer_ToContiguous() for non-contiguous arrays.
+
 - Issue #15456: Fix code __sizeof__ after #12399 change.
   Patch by Serhiy Storchaka.
 

Modules/_testbuffer.c

@@ -2397,6 +2397,49 @@ get_contiguous(PyObject *self, PyObject *args)
     return PyMemoryView_GetContiguous(obj, (int)type, ord);
 }
 
+/* PyBuffer_ToContiguous() */
+static PyObject *
+py_buffer_to_contiguous(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    PyObject *order;
+    PyObject *ret = NULL;
+    int flags;
+    char ord;
+    Py_buffer view;
+    char *buf = NULL;
+
+    if (!PyArg_ParseTuple(args, "OOi", &obj, &order, &flags)) {
+        return NULL;
+    }
+
+    if (PyObject_GetBuffer(obj, &view, flags) < 0) {
+        return NULL;
+    }
+
+    ord = get_ascii_order(order);
+    if (ord == CHAR_MAX) {
+        goto out;
+    }
+
+    buf = PyMem_Malloc(view.len);
+    if (buf == NULL) {
+        PyErr_NoMemory();
+        goto out;
+    }
+
+    if (PyBuffer_ToContiguous(buf, &view, view.len, ord) < 0) {
+        goto out;
+    }
+
+    ret = PyBytes_FromStringAndSize(buf, view.len);
+
+out:
+    PyBuffer_Release(&view);
+    PyMem_XFree(buf);
+    return ret;
+}
+
 static int
 fmtcmp(const char *fmt1, const char *fmt2)
 {
@@ -2734,6 +2777,7 @@ static struct PyMethodDef _testbuffer_functions[] = {
     {"get_pointer", get_pointer, METH_VARARGS, NULL},
     {"get_sizeof_void_p", (PyCFunction)get_sizeof_void_p, METH_NOARGS, NULL},
     {"get_contiguous", get_contiguous, METH_VARARGS, NULL},
+    {"py_buffer_to_contiguous", py_buffer_to_contiguous, METH_VARARGS, NULL},
     {"is_contiguous", is_contiguous, METH_VARARGS, NULL},
     {"cmp_contig", cmp_contig, METH_VARARGS, NULL},
     {NULL, NULL}

Objects/abstract.c

@@ -445,62 +445,6 @@ _Py_add_one_to_index_C(int nd, Py_ssize_t *index, const Py_ssize_t *shape)
     }
 }
 
-  /* view is not checked for consistency in either of these.  It is
-     assumed that the size of the buffer is view->len in
-     view->len / view->itemsize elements.
-  */
-
-int
-PyBuffer_ToContiguous(void *buf, Py_buffer *view, Py_ssize_t len, char fort)
-{
-    int k;
-    void (*addone)(int, Py_ssize_t *, const Py_ssize_t *);
-    Py_ssize_t *indices, elements;
-    char *dest, *ptr;
-
-    if (len > view->len) {
-        len = view->len;
-    }
-
-    if (PyBuffer_IsContiguous(view, fort)) {
-        /* simplest copy is all that is needed */
-        memcpy(buf, view->buf, len);
-        return 0;
-    }
-
-    /* Otherwise a more elaborate scheme is needed */
-
-    /* XXX(nnorwitz): need to check for overflow! */
-    indices = (Py_ssize_t *)PyMem_Malloc(sizeof(Py_ssize_t)*(view->ndim));
-    if (indices == NULL) {
-        PyErr_NoMemory();
-        return -1;
-    }
-    for (k=0; k<view->ndim;k++) {
-        indices[k] = 0;
-    }
-
-    if (fort == 'F') {
-        addone = _Py_add_one_to_index_F;
-    }
-    else {
-        addone = _Py_add_one_to_index_C;
-    }
-    dest = buf;
-    /* XXX : This is not going to be the fastest code in the world
-             several optimizations are possible.
-     */
-    elements = len / view->itemsize;
-    while (elements--) {
-        addone(view->ndim, indices, view->shape);
-        ptr = PyBuffer_GetPointer(view, indices);
-        memcpy(dest, ptr, view->itemsize);
-        dest += view->itemsize;
-    }
-    PyMem_Free(indices);
-    return 0;
-}
-
 int
 PyBuffer_FromContiguous(Py_buffer *view, void *buf, Py_ssize_t len, char fort)
 {

Objects/bytesobject.c

@@ -2591,7 +2591,6 @@ PyBytes_FromObject(PyObject *x)
         new = PyBytes_FromStringAndSize(NULL, view.len);
         if (!new)
             goto fail;
-        /* XXX(brett.cannon): Better way to get to internal buffer? */
         if (PyBuffer_ToContiguous(((PyBytesObject *)new)->ob_sval,
                                   &view, view.len, 'C') < 0)
             goto fail;

Objects/memoryobject.c

@@ -438,15 +438,17 @@ init_fortran_strides_from_shape(Py_buffer *view)
         view->strides[i] = view->strides[i-1] * view->shape[i-1];
 }
 
-/* Copy src to a C-contiguous representation. Assumptions:
+/* Copy src to a contiguous representation. order is one of 'C', 'F' (Fortran)
+   or 'A' (Any). Assumptions: src has PyBUF_FULL information, src->ndim >= 1,
    len(mem) == src->len. */
 static int
-buffer_to_c_contiguous(char *mem, Py_buffer *src)
+buffer_to_contiguous(char *mem, Py_buffer *src, char order)
 {
     Py_buffer dest;
     Py_ssize_t *strides;
     int ret;
 
+    assert(src->ndim >= 1);
     assert(src->shape != NULL);
     assert(src->strides != NULL);
 
@@ -456,12 +458,22 @@ buffer_to_c_contiguous(char *mem, Py_buffer *src)
         return -1;
     }
 
-    /* initialize dest as a C-contiguous buffer */
+    /* initialize dest */
     dest = *src;
     dest.buf = mem;
-    /* shape is constant and shared */
+    /* shape is constant and shared: the logical representation of the
+       array is unaltered. */
+
+    /* The physical representation determined by strides (and possibly
+       suboffsets) may change. */
     dest.strides = strides;
+    if (order == 'C' || order == 'A') {
         init_strides_from_shape(&dest);
+    }
+    else {
+        init_fortran_strides_from_shape(&dest);
+    }
+
     dest.suboffsets = NULL;
 
     ret = copy_buffer(&dest, src);
@@ -921,6 +933,57 @@ memory_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)
 }
 
 
+/****************************************************************************/
+/*                         Previously in abstract.c                         */
+/****************************************************************************/
+
+typedef struct {
+    Py_buffer view;
+    Py_ssize_t array[1];
+} Py_buffer_full;
+
+int
+PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order)
+{
+    Py_buffer_full *fb = NULL;
+    int ret;
+
+    assert(order == 'C' || order == 'F' || order == 'A');
+
+    if (len != src->len) {
+        PyErr_SetString(PyExc_ValueError,
+            "PyBuffer_ToContiguous: len != view->len");
+        return -1;
+    }
+
+    if (PyBuffer_IsContiguous(src, order)) {
+        memcpy((char *)buf, src->buf, len);
+        return 0;
+    }
+
+    /* buffer_to_contiguous() assumes PyBUF_FULL */
+    fb = PyMem_Malloc(sizeof *fb + 3 * src->ndim * (sizeof *fb->array));
+    if (fb == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
+    fb->view.ndim = src->ndim;
+    fb->view.shape = fb->array;
+    fb->view.strides = fb->array + src->ndim;
+    fb->view.suboffsets = fb->array + 2 * src->ndim;
+
+    init_shared_values(&fb->view, src);
+    init_shape_strides(&fb->view, src);
+    init_suboffsets(&fb->view, src);
+
+    src = &fb->view;
+
+    ret = buffer_to_contiguous(buf, src, order);
+    PyMem_Free(fb);
+    return ret;
+}
+
+
 /****************************************************************************/
 /*                           Release/GC management                          */
 /****************************************************************************/
@@ -1889,7 +1952,7 @@ memory_tobytes(PyMemoryViewObject *self, PyObject *dummy)
     if (bytes == NULL)
         return NULL;
 
-    if (buffer_to_c_contiguous(PyBytes_AS_STRING(bytes), src) < 0) {
+    if (buffer_to_contiguous(PyBytes_AS_STRING(bytes), src, 'C') < 0) {
         Py_DECREF(bytes);
         return NULL;
     }
@@ -2423,7 +2486,7 @@ memory_hash(PyMemoryViewObject *self)
                 PyErr_NoMemory();
                 return -1;
             }
-            if (buffer_to_c_contiguous(mem, view) < 0) {
+            if (buffer_to_contiguous(mem, view, 'C') < 0) {
                 PyMem_Free(mem);
                 return -1;
             }