cleaned up exported C-API of array.array object

Cython/Compiler/Optimize.py

@@ -1982,6 +1982,8 @@ class OptimizeBuiltinCalls(Visitor.MethodDispatcherTransform):
         Builtin.frozenset_type : "PySet_Size",
         }.get
 
+    _ext_types_with_pysize = set(["cpython.array.array"])
+
     def _handle_simple_function_len(self, node, pos_args):
         """Replace len(char*) by the equivalent call to strlen() and
         len(known_builtin_type) by an equivalent C-API call.
@@ -2001,6 +2003,11 @@ class OptimizeBuiltinCalls(Visitor.MethodDispatcherTransform):
         elif arg.type.is_pyobject:
             cfunc_name = self._map_to_capi_len_function(arg.type)
             if cfunc_name is None:
+                arg_type = arg.type
+                if ((arg_type.is_extension_type or arg_type.is_builtin_type)
+                    and arg_type.entry.qualified_name in self._ext_types_with_pysize):
+                    cfunc_name = 'Py_SIZE'
+                else:
                     return node
             arg = arg.as_none_safe_node(
                 "object of type 'NoneType' has no len()")

Cython/Includes/cpython/array.pxd

@@ -17,11 +17,11 @@
     ...     array.array[double] a = arg1
     ...     a[i] += dx
   
-  Fast C-level new_array(_zeros), resize_array, copy_array, .length,
+  Fast C-level new_array(_zeros), resize_array, copy_array, Py_SIZE(obj),
   zero_array
   
     cdef array.array[double] k = array.copy(d) 
-    cdef array.array[double] n = array.array(d, d.length * 2 ) 
+    cdef array.array[double] n = array.array(d, Py_SIZE(d) * 2 )
     cdef array.array[double] m = array.zeros_like(FLOAT_TEMPLATE)
     array.resize(f, 200000)
   
@@ -50,6 +50,7 @@ from libc cimport stdlib
 from libc.string cimport strcat, strncat, \
     memset, memchr, memcmp, memcpy, memmove
 
+from cpython.object cimport Py_SIZE
 from cpython.ref cimport PyTypeObject, Py_TYPE
 from cpython.exc cimport PyErr_BadArgument
 
@@ -63,27 +64,29 @@ cdef extern from *:  # Hard-coded utility code hack.
             GETF getitem    # PyObject * (*getitem)(struct arrayobject *, Py_ssize_t);
             SETF setitem    # int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *);
 
+    ctypedef union __data_union:
+        # views of ob_item:
+        float* as_floats        # direct float pointer access to buffer
+        double* as_doubles      # double ...
+        int*    as_ints
+        unsigned int *as_uints
+        unsigned char *as_uchars
+        signed char *as_schars
+        char *as_chars
+        unsigned long *as_ulongs
+        long *as_longs
+        short *as_shorts
+        unsigned short *as_ushorts
+        Py_UNICODE *as_pyunicodes
+        void *as_voidptr
+
     ctypedef class array.array [object arrayobject]:
         cdef __cythonbufferdefaults__ = {'ndim' : 1, 'mode':'c'}
         
         cdef:
-            Py_ssize_t length       # == ob_size (by union)
+            Py_ssize_t ob_size
             arraydescr* ob_descr    # struct arraydescr *ob_descr;
-
-            # views of ob_item:
-            float* _f               # direct float pointer access to buffer
-            double* _d              # double ...
-            int*    _i
-            unsigned *_I
-            unsigned char *_B
-            signed char *_b
-            char *_c
-            unsigned long *_L
-            long *_l
-            short *_h
-            unsigned short *_H
-            Py_UNICODE *_u
-            void* _v
+            __data_union data
 
         def __getbuffer__(array self, Py_buffer* info, int flags):
             # This implementation of getbuffer is geared towards Cython
@@ -93,7 +96,7 @@ cdef extern from *:  # Hard-coded utility code hack.
             cdef unsigned rows, columns, itemsize
             
             info.suboffsets = NULL
-            info.buf = self._c
+            info.buf = self.data.as_chars
             info.readonly = 0
             info.ndim = 1
             info.itemsize = itemsize = self.ob_descr.itemsize   # e.g. sizeof(float)
@@ -101,7 +104,7 @@ cdef extern from *:  # Hard-coded utility code hack.
             info.strides = <Py_ssize_t*> \
                            stdlib.malloc(sizeof(Py_ssize_t) * info.ndim * 2 + 2)
             info.shape = info.strides + 1
-            info.shape[0] = self.length            # number of items
+            info.shape[0] = Py_SIZE(self)            # number of items
             info.strides[0] = info.itemsize
 
             info.format = <char*>(info.strides + 2 * info.ndim)
@@ -115,8 +118,7 @@ cdef extern from *:  # Hard-coded utility code hack.
             #if sizeof(npy_intp) != sizeof(Py_ssize_t):
             stdlib.free(info.strides)
         
-    array newarrayobject(PyTypeObject* type, Py_ssize_t size,
-                              arraydescr *descr)
+    array newarrayobject(PyTypeObject* type, Py_ssize_t size, arraydescr *descr)
 
     # fast resize/realloc
     # not suitable for small increments; reallocation 'to the point'
@@ -132,14 +134,14 @@ cdef inline array clone(array template, Py_ssize_t length, bint zero):
     cdef array op
     op = newarrayobject(Py_TYPE(template), length, template.ob_descr)
     if zero and op is not None:
-        memset(op._c, 0, length * op.ob_descr.itemsize)
+        memset(op.data.as_chars, 0, length * op.ob_descr.itemsize)
     return op
 
 cdef inline array copy(array self):
     """ make a copy of an array. """
     cdef array op
-    op = newarrayobject(Py_TYPE(self), self.length, self.ob_descr)
-    memcpy(op._c, self._c, op.length * op.ob_descr.itemsize)
+    op = newarrayobject(Py_TYPE(self), Py_SIZE(self), self.ob_descr)
+    memcpy(op.data.as_chars, self.data.as_chars, Py_SIZE(op) * op.ob_descr.itemsize)
     return op
 
 cdef inline int extend_buffer(array self, char* stuff, Py_ssize_t n):
@@ -147,18 +149,18 @@ cdef inline int extend_buffer(array self, char* stuff, Py_ssize_t n):
     (e.g. of same array type)
     n: number of elements (not number of bytes!) """
     cdef Py_ssize_t itemsize = self.ob_descr.itemsize
-    cdef Py_ssize_t orgsize = self.length
+    cdef Py_ssize_t orgsize = Py_SIZE(self)
     if resize_smart(self, orgsize + n) == -1:
         return -1
-    memcpy(self._c + orgsize * itemsize, stuff, n * itemsize)
+    memcpy(self.data.as_chars + orgsize * itemsize, stuff, n * itemsize)
 
 cdef inline int extend(array self, array other):
     """ extend array with data from another array; types must match. """
     if self.ob_descr.typecode != self.ob_descr.typecode:
         PyErr_BadArgument()
         return -1
-    return extend_buffer(self, other._c, other.length)
+    return extend_buffer(self, other.data.as_chars, Py_SIZE(other))
 
 cdef inline void zero(array op):
     """ set all elements of array to zero. """
-    memset(op._c, 0, op.length * op.ob_descr.itemsize)
+    memset(op.data.as_chars, 0, Py_SIZE(op) * op.ob_descr.itemsize)

Cython/Includes/cpython/object.pxd

@@ -284,3 +284,4 @@ cdef extern from "Python.h":
     # itself if the object is already an iterator. Raises TypeError
     # and returns NULL if the object cannot be iterated.
 
+    Py_ssize_t Py_SIZE(object o)

Cython/Utility/arrayarray.h

@@ -31,32 +31,26 @@ typedef struct arraydescr {
 
 typedef struct arrayobject {
     PyObject_HEAD
-    union {
-        Py_ssize_t ob_size, length;
-    };
+    Py_ssize_t ob_size;
     union {
         char *ob_item;
-        float *_f;
-        double *_d;
-        int *_i;
-        unsigned *_I;
-        unsigned char *_B;
-        signed char *_b;
-        char *_c;
-        unsigned long *_L;
-        long *_l;
-        short *_h;
-        unsigned short *_H;
-        Py_UNICODE *_u;
-        void *_v;
-    };
-#if PY_VERSION_HEX >= 0x02040000
+        float *as_floats;
+        double *as_doubles;
+        int *as_ints;
+        unsigned int *as_uints;
+        unsigned char *as_uchars;
+        signed char *as_schars;
+        char *as_chars;
+        unsigned long *as_ulongs;
+        long *as_longs;
+        short *as_shorts;
+        unsigned short *as_ushorts;
+        Py_UNICODE *as_pyunicodes;
+        void *as_voidptr;
+    } data;
     Py_ssize_t allocated;
-#endif
     struct arraydescr *ob_descr;
-#if PY_VERSION_HEX >= 0x02040000
     PyObject *weakreflist; /* List of weak references */
-#endif
 #if PY_VERSION_HEX >= 0x03000000
         int ob_exports;  /* Number of exported buffers */
 #endif
@@ -76,7 +70,7 @@ static CYTHON_INLINE PyObject * newarrayobject(PyTypeObject *type, Py_ssize_t si
     }
 
     nbytes = size * descr->itemsize;
-    /* Check for overflow */
+    // Check for overflow
     if (nbytes / descr->itemsize != (size_t)size) {
         return PyErr_NoMemory();
     }
@@ -85,17 +79,15 @@ static CYTHON_INLINE PyObject * newarrayobject(PyTypeObject *type, Py_ssize_t si
         return NULL;
     }
     op->ob_descr = descr;
-#if !( PY_VERSION_HEX < 0x02040000 )
     op->allocated = size;
     op->weakreflist = NULL;
-#endif
-    Py_SIZE(op) = size;
+    op->ob_size = size;
     if (size <= 0) {
-        op->ob_item = NULL;
+        op->data.ob_item = NULL;
     }
     else {
-        op->ob_item = PyMem_NEW(char, nbytes);
-        if (op->ob_item == NULL) {
+        op->data.ob_item = PyMem_NEW(char, nbytes);
+        if (op->data.ob_item == NULL) {
             Py_DECREF(op);
             return PyErr_NoMemory();
         }
@@ -109,26 +101,23 @@ PyObject* newarrayobject(PyTypeObject *type, Py_ssize_t size,
 
 // fast resize (reallocation to the point)
 // not designed for filing small increments (but for fast opaque array apps)
-static int resize(arrayobject *self, Py_ssize_t n) {
-    void *item= (void*) self->ob_item;
-    PyMem_Resize(item, char, (size_t)(n * self->ob_descr->itemsize));
-    if (item == NULL) {
+static CYTHON_INLINE int resize(arrayobject *self, Py_ssize_t n) {
+    void *items = (void*) self->data.ob_item;
+    PyMem_Resize(items, char, (size_t)(n * self->ob_descr->itemsize));
+    if (items == NULL) {
         PyErr_NoMemory();
         return -1;
     }    
-    self->ob_item = (char*) item;
+    self->data.ob_item = (char*) items;
     self->ob_size = n;
-#if PY_VERSION_HEX >= 0x02040000
     self->allocated = n;
-#endif
     return 0;
 }
 
 // suitable for small increments; over allocation 50% ;
 // Remains non-smart in Python 2.3- ; but exists for compatibility
-static int resize_smart(arrayobject *self, Py_ssize_t n) {
-#if PY_VERSION_HEX >= 0x02040000
-    void *item = (void*) self->ob_item;
+static CYTHON_INLINE int resize_smart(arrayobject *self, Py_ssize_t n) {
+    void *items = (void*) self->data.ob_item;
     Py_ssize_t newsize;
     if (n < self->allocated) {
         if (n*4 > self->allocated) {
@@ -137,18 +126,15 @@ static int resize_smart(arrayobject *self, Py_ssize_t n) {
         }
     }
     newsize = n  * 3 / 2 + 1;
-    PyMem_Resize(item, char, (size_t)(newsize * self->ob_descr->itemsize));
-    if (item == NULL) {
+    PyMem_Resize(items, char, (size_t)(newsize * self->ob_descr->itemsize));
+    if (items == NULL) {
         PyErr_NoMemory();
         return -1;
     }    
-    self->ob_item = (char*) item;
+    self->data.ob_item = (char*) items;
     self->ob_size = n;
     self->allocated = newsize;
     return 0;
-#else
-    return resize(self, n)   /* Python 2.3 has no 'allocated' */
-#endif
 }
 
 #endif

runtests.py

@@ -83,10 +83,6 @@ def update_numpy_extension(ext):
     import numpy
     ext.include_dirs.append(numpy.get_include())
 
-def update_pyarray_extension(ext):
-    # See http://gcc.gnu.org/onlinedocs/gcc/Unnamed-Fields.html#Unnamed-Fields
-    ext.extra_compile_args.append('-fms-extensions')
-
 def update_openmp_extension(ext):
     ext.openmp = True
     language = ext.language
@@ -162,7 +158,6 @@ EXCLUDE_EXT = object()
 
 EXT_EXTRAS = {
     'tag:numpy' : update_numpy_extension,
-    'tag:array' : update_pyarray_extension,
     'tag:openmp': update_openmp_extension,
 }
 

tests/run/pyarray.pyx

@@ -15,7 +15,7 @@ def test_len(a):
     >>> assert len(a) == test_len(a)
     """
     cdef array.array ca = a  # for C-fast array usage
-    return ca.length
+    return len(ca)
 
 def test_copy(a):
     """
@@ -42,14 +42,14 @@ def test_fast_access(a):
     
     cdef float value
     with nogil:
-        value = ca._f[1]
+        value = ca.data.as_floats[1]
     assert value == 2.0, value
 
     #assert ca._c[:5] == b'\x00\x00\x80?\x00', repr(ca._c[:5])
 
     with nogil:
-        ca._f[1] += 2.0
-    assert ca._f[1] == 4.0
+        ca.data.as_floats[1] += 2.0
+    assert ca.data.as_floats[1] == 4.0
 
 
 def test_fast_buffer_access(a):
@@ -77,7 +77,7 @@ def test_new_zero(a):
     array('f', [0.0, 0.0, 0.0])
     """
     cdef array.array cb = array.clone(a, len(a), True)
-    assert cb.length == len(a)
+    assert len(cb) == len(a)
     return cb
 
 
@@ -101,9 +101,9 @@ def test_resize(a):
     cdef array.array cb = array.copy(a)
     array.resize(cb, 10)
     for i in range(10):
-        cb._f[i] = i
-    assert cb.length == 10
-    assert cb[9] == cb[-1] == cb._f[9] == 9
+        cb.data.as_floats[i] = i
+    assert len(cb) == 10
+    assert cb[9] == cb[-1] == cb.data.as_floats[9] == 9
 
 def test_buffer():
     """
@@ -158,7 +158,7 @@ def test_likes(a):
     """
     cdef array.array z = array.clone(a, len(a), True)
     cdef array.array e = array.clone(a, len(a), False)
-    assert e.length == len(a)
+    assert len(e) == len(a)
     return z
 
 def test_extend_buffer():
@@ -172,6 +172,6 @@ def test_extend_buffer():
     s[1] = 5077
     array.extend_buffer(ca, <char*> &s, 2)
 
-    assert ca._L[3] == 5077
+    assert ca.data.as_ulongs[3] == 5077
     assert len(ca) == 4
     return ca