Stopgap support for numpy 1.7

NumPy is starting to seriously deprecating access to the member fields
in an ndarray (it was always frowned upon, but now it is starting to
become enforced). To support the large body of Cython code out there
accessing these fields (arr.shape[0] and so on), we special-case
PyArrayObject in Cython, with special knowledge of the NumPy API.

Ideally, we may introduce features in Cython in the future that allows
specifying this kind of magic with syntax in pxd files, and then we can
move away from special-casing NumPy.

Cython/Compiler/ExprNodes.py

@@ -4428,6 +4428,17 @@ class AttributeNode(ExprNode):
                     # method of an extension type, so we treat it like a Python
                     # attribute.
                     pass
+        # NumPy hack
+        if obj_type.is_extension_type and obj_type.objstruct_cname == 'PyArrayObject':
+            from NumpySupport import numpy_transform_attribute_node
+            replacement_node = numpy_transform_attribute_node(self)
+            # Since we can't actually replace our node yet, we only grasp its
+            # type, and then the replacement happens in
+            # AnalyseExpresssionsTransform...
+            self.type = replacement_node.type
+            if replacement_node is not self:
+                return
+        
         # If we get here, the base object is not a struct/union/extension
         # type, or it is an extension type and the attribute is either not
         # declared or is declared as a Python method. Treat it as a Python

Cython/Compiler/NumpySupport.py

+# The hacks that are specific for NumPy. These were introduced because
+# the NumPy ABI changed so that the shape, ndim, strides, etc. fields were
+# no longer available, however the use of these were so entrenched in
+# Cython codes
+
+import PyrexTypes
+import ExprNodes
+from StringEncoding import EncodedString
+
+
+def numpy_transform_attribute_node(node):
+    assert isinstance(node, ExprNodes.AttributeNode)
+
+    if node.obj.type.objstruct_cname != 'PyArrayObject':
+        return node
+
+    pos = node.pos
+    numpy_pxd_scope = node.obj.entry.type.scope.parent_scope
+        
+    def macro_call_node(numpy_macro_name):
+        array_node = node.obj
+        func_entry = numpy_pxd_scope.entries[numpy_macro_name]
+        function_name_node = ExprNodes.NameNode(
+            name=EncodedString(numpy_macro_name),
+            pos=pos,
+            entry=func_entry,
+            is_called=1,
+            type=func_entry.type,
+            cf_maybe_null=False,
+            cf_is_null=False)
+        
+        call_node = ExprNodes.SimpleCallNode(
+            pos=pos,
+            function=function_name_node,
+            name=EncodedString(numpy_macro_name),
+            args=[array_node],
+            type=func_entry.type.return_type,
+            analysed=True)
+        return call_node
+        
+    
+    if node.attribute == u'ndim':
+        result = macro_call_node(u'PyArray_NDIM')
+    elif node.attribute == u'data':
+        call_node = macro_call_node(u'PyArray_DATA')
+        cast_node = ExprNodes.TypecastNode(pos,
+                                           type=PyrexTypes.c_char_ptr_type,
+                                           operand=call_node)
+        result = cast_node
+    elif node.attribute == u'shape':
+        result = macro_call_node(u'PyArray_DIMS')
+    elif node.attribute == u'strides':
+        result = macro_call_node(u'PyArray_STRIDES')
+    else:
+        result = node
+    return result

Cython/Compiler/ParseTreeTransforms.py

@@ -1744,6 +1744,7 @@ if VALUE is not None:
 
 
 class AnalyseExpressionsTransform(CythonTransform):
+    # Also handles NumPy
 
     def visit_ModuleNode(self, node):
         self.env_stack = [node.scope]
@@ -1785,9 +1786,18 @@ class AnalyseExpressionsTransform(CythonTransform):
         elif node.memslice_ellipsis_noop:
             # memoryviewslice[...] expression, drop the IndexNode
             node = node.base
-
         return node
 
+    def visit_AttributeNode(self, node):
+        # Note: Expression analysis for attributes has already happened
+        # at this point (by recursive calls starting from FuncDefNode)
+        #print node.dump()
+        #return node
+        type = node.obj.type
+        if type.is_extension_type and type.objstruct_cname == 'PyArrayObject':
+            from NumpySupport import numpy_transform_attribute_node
+            node = numpy_transform_attribute_node(node)
+        return node
 
 class FindInvalidUseOfFusedTypes(CythonTransform):
 

Cython/Compiler/Pipeline.py

@@ -188,6 +188,7 @@ def create_pipeline(context, mode, exclude_classes=()):
         _check_c_declarations,
         InlineDefNodeCalls(context),
         AnalyseExpressionsTransform(context),
+        # AnalyseExpressionsTransform also contains the NumPy-specific support
         FindInvalidUseOfFusedTypes(context),
         CreateClosureClasses(context),  ## After all lookups and type inference
         ExpandInplaceOperators(context),

Cython/Includes/numpy.pxd

@@ -151,6 +151,9 @@ cdef extern from "numpy/arrayobject.h":
 
     ctypedef void (*PyArray_VectorUnaryFunc)(void *, void *, npy_intp, void *,  void *)
 
+    ctypedef struct PyArray_Descr:
+        pass
+    
     ctypedef class numpy.dtype [object PyArray_Descr]:
         # Use PyDataType_* macros when possible, however there are no macros
         # for accessing some of the fields, so some are defined. Please
@@ -177,15 +180,11 @@ cdef extern from "numpy/arrayobject.h":
     ctypedef class numpy.ndarray [object PyArrayObject]:
         cdef __cythonbufferdefaults__ = {"mode": "strided"}
 
-        cdef:
-            # Only taking a few of the most commonly used and stable fields.
-            # One should use PyArray_* macros instead to access the C fields.
-            char *data
-            int ndim "nd"
-            npy_intp *shape "dimensions"
-            npy_intp *strides
-            dtype descr
-            PyObject* base
+        # Note: The fields are no longer defined, please use accessor
+        # functions. Cython special-cases/hacks the data, ndim, shape
+        # and stride attributes of the ndarray to use accessor
+        # functions for backwards compatability and convenience.
+
 
         # Note: This syntax (function definition in pxd files) is an
         # experimental exception made for __getbuffer__ and __releasebuffer__
@@ -236,7 +235,7 @@ cdef extern from "numpy/arrayobject.h":
 
             cdef int t
             cdef char* f = NULL
-            cdef dtype descr = self.descr
+            cdef dtype descr = get_array_dtype(self)
             cdef list stack
             cdef int offset
 
@@ -376,20 +375,29 @@ cdef extern from "numpy/arrayobject.h":
     bint PyArray_ISWRITEABLE(ndarray m)
     bint PyArray_ISALIGNED(ndarray m)
 
-    int PyArray_NDIM(ndarray)
+    int PyArray_NDIM(ndarray) nogil
     bint PyArray_ISONESEGMENT(ndarray)
     bint PyArray_ISFORTRAN(ndarray)
     int PyArray_FORTRANIF(ndarray)
 
-    void* PyArray_DATA(ndarray)
-    char* PyArray_BYTES(ndarray)
-    npy_intp* PyArray_DIMS(ndarray)
-    npy_intp* PyArray_STRIDES(ndarray)
-    npy_intp PyArray_DIM(ndarray, size_t)
-    npy_intp PyArray_STRIDE(ndarray, size_t)
+    void* PyArray_DATA(ndarray) nogil
+    char* PyArray_BYTES(ndarray) nogil
+    npy_intp* PyArray_DIMS(ndarray) nogil
+    npy_intp* PyArray_STRIDES(ndarray) nogil
+    npy_intp PyArray_DIM(ndarray, size_t) nogil
+    npy_intp PyArray_STRIDE(ndarray, size_t) nogil
+
+    # The two functions below return borrowed references and should
+    # be used with care; often you will want to use get_array_base
+    # or get_array_dtype (define below) instead from Cython.
+    PyObject* PyArray_BASE(ndarray)
+    # Cython API of the function below might change! PyArray_DESCR
+    # actually returns PyArray_Descr* == pointer-version of dtype,
+    # which appears to be difficult to declare properly in Cython;
+    # protect it with trailing underscore for now just to avoid having
+    # user code depend on it without reading this note.
+    PyArray_Descr * PyArray_DESCR_ "PyArray_DESCR"(ndarray)
 
-    # object PyArray_BASE(ndarray) wrong refcount semantics
-    # dtype PyArray_DESCR(ndarray) wrong refcount semantics
     int PyArray_FLAGS(ndarray)
     npy_intp PyArray_ITEMSIZE(ndarray)
     int PyArray_TYPE(ndarray arr)
@@ -961,18 +969,34 @@ cdef extern from "numpy/ufuncobject.h":
     void import_ufunc()
 
 
-cdef inline void set_array_base(ndarray arr, object base):
-     cdef PyObject* baseptr
-     if base is None:
-         baseptr = NULL
-     else:
-         Py_INCREF(base) # important to do this before decref below!
-         baseptr = <PyObject*>base
-     Py_XDECREF(arr.base)
-     arr.base = baseptr
+# The ability to set the base field of an ndarray seems to be
+# deprecated in NumPy 1.7 (no PyArray_SET_BASE seems to be
+# available). Remove this support and see who complains and how their
+# case could be fixed in 1.7...
+#
+#cdef inline void set_array_base(ndarray arr, object base):
+#     cdef PyObject* baseptr
+#     if base is None:
+#         baseptr = NULL
+#     else:
+#         Py_INCREF(base) # important to do this before decref below!
+#         baseptr = <PyObject*>base
+#     Py_XDECREF(arr.base)
+#     arr.base = baseptr
 
 cdef inline object get_array_base(ndarray arr):
-    if arr.base is NULL:
+    cdef PyObject *pobj = PyArray_BASE(arr)
+    if pobj != NULL:
+        obj = <object>pobj
+        Py_INCREF(obj)
+        return obj
+    else:
         return None
+
+cdef inline dtype get_array_dtype(ndarray arr):
+    if PyArray_DESCR_(arr) != NULL:
+        obj = <object>PyArray_DESCR_(arr)
+        Py_INCREF(obj)
+        return obj
     else:
-        return <object>arr.base
+        return None

tests/run/numpy_attributes.pyx

+# tag: numpy
+
+
+import numpy as np
+cimport numpy as np
+
+
+def f():
+    """
+    >>> f()
+    ndim 2
+    data 1
+    shape 3 2
+    shape[1] 2
+    strides 16 8
+    """
+    cdef np.ndarray x = np.ones((3, 2), dtype=np.int64)
+    cdef int i
+    cdef Py_ssize_t j, k
+    cdef char *p
+    # todo: int * p: 23:13: Cannot assign type 'char *' to 'int *'
+
+    with nogil:
+        i = x.ndim
+    print 'ndim', i
+    
+    with nogil:
+        p = x.data
+    print 'data', (<np.int64_t*>p)[0]
+
+    with nogil:
+        j = x.shape[0]
+        k = x.shape[1]
+    print 'shape', j, k
+    # Check that non-typical uses still work
+    cdef np.npy_intp *shape
+    with nogil:
+        shape = x.shape + 1
+    print 'shape[1]', shape[0]
+
+    with nogil:
+        j = x.strides[0]
+        k = x.strides[1]
+    print 'strides', j, k
+