Merge branch '_newnumpyapi' of https://github.com/dagss/cython into release

Cython/Compiler/ExprNodes.py

@@ -1363,16 +1363,9 @@ class NameNode(AtomicExprNode):
     allow_null = False
     nogil = False
 
-    def create_analysed_rvalue(pos, env, entry):
-        node = NameNode(pos)
-        node.analyse_types(env, entry=entry)
-        return node
-
     def as_cython_attribute(self):
         return self.cython_attribute
 
-    create_analysed_rvalue = staticmethod(create_analysed_rvalue)
-
     def type_dependencies(self, env):
         if self.entry is None:
             self.entry = env.lookup(self.name)
@@ -4435,6 +4428,17 @@ class AttributeNode(ExprNode):
                     # method of an extension type, so we treat it like a Python
                     # attribute.
                     pass
+        # NumPy hack
+        if obj_type.is_extension_type and obj_type.objstruct_cname == 'PyArrayObject':
+            from NumpySupport import numpy_transform_attribute_node
+            replacement_node = numpy_transform_attribute_node(self)
+            # Since we can't actually replace our node yet, we only grasp its
+            # type, and then the replacement happens in
+            # AnalyseExpresssionsTransform...
+            self.type = replacement_node.type
+            if replacement_node is not self:
+                return
+        
         # If we get here, the base object is not a struct/union/extension
         # type, or it is an extension type and the attribute is either not
         # declared or is declared as a Python method. Treat it as a Python

Cython/Compiler/NumpySupport.py

+# The hacks that are specific for NumPy. These were introduced because
+# the NumPy ABI changed so that the shape, ndim, strides, etc. fields were
+# no longer available, however the use of these were so entrenched in
+# Cython codes
+
+import PyrexTypes
+import ExprNodes
+from StringEncoding import EncodedString
+
+
+def numpy_transform_attribute_node(node):
+    assert isinstance(node, ExprNodes.AttributeNode)
+
+    if node.obj.type.objstruct_cname != 'PyArrayObject':
+        return node
+
+    pos = node.pos
+    numpy_pxd_scope = node.obj.entry.type.scope.parent_scope
+        
+    def macro_call_node(numpy_macro_name):
+        array_node = node.obj
+        func_entry = numpy_pxd_scope.entries[numpy_macro_name]
+        function_name_node = ExprNodes.NameNode(
+            name=EncodedString(numpy_macro_name),
+            pos=pos,
+            entry=func_entry,
+            is_called=1,
+            type=func_entry.type,
+            cf_maybe_null=False,
+            cf_is_null=False)
+        
+        call_node = ExprNodes.SimpleCallNode(
+            pos=pos,
+            function=function_name_node,
+            name=EncodedString(numpy_macro_name),
+            args=[array_node],
+            type=func_entry.type.return_type,
+            analysed=True)
+        return call_node
+        
+    
+    if node.attribute == u'ndim':
+        result = macro_call_node(u'PyArray_NDIM')
+    elif node.attribute == u'data':
+        call_node = macro_call_node(u'PyArray_DATA')
+        cast_node = ExprNodes.TypecastNode(pos,
+                                           type=PyrexTypes.c_char_ptr_type,
+                                           operand=call_node)
+        result = cast_node
+    elif node.attribute == u'shape':
+        result = macro_call_node(u'PyArray_DIMS')
+    elif node.attribute == u'strides':
+        result = macro_call_node(u'PyArray_STRIDES')
+    else:
+        result = node
+    return result

Cython/Compiler/ParseTreeTransforms.py

@@ -1744,6 +1744,7 @@ if VALUE is not None:
 
 
 class AnalyseExpressionsTransform(CythonTransform):
+    # Also handles NumPy
 
     def visit_ModuleNode(self, node):
         self.env_stack = [node.scope]
@@ -1785,9 +1786,18 @@ class AnalyseExpressionsTransform(CythonTransform):
         elif node.memslice_ellipsis_noop:
             # memoryviewslice[...] expression, drop the IndexNode
             node = node.base
-
         return node
 
+    def visit_AttributeNode(self, node):
+        # Note: Expression analysis for attributes has already happened
+        # at this point (by recursive calls starting from FuncDefNode)
+        #print node.dump()
+        #return node
+        type = node.obj.type
+        if type.is_extension_type and type.objstruct_cname == 'PyArrayObject':
+            from NumpySupport import numpy_transform_attribute_node
+            node = numpy_transform_attribute_node(node)
+        return node
 
 class FindInvalidUseOfFusedTypes(CythonTransform):
 

Cython/Compiler/Pipeline.py

@@ -188,6 +188,7 @@ def create_pipeline(context, mode, exclude_classes=()):
         _check_c_declarations,
         InlineDefNodeCalls(context),
         AnalyseExpressionsTransform(context),
+        # AnalyseExpressionsTransform also contains the NumPy-specific support
         FindInvalidUseOfFusedTypes(context),
         CreateClosureClasses(context),  ## After all lookups and type inference
         ExpandInplaceOperators(context),

Cython/Includes/numpy.pxd

@@ -151,6 +151,9 @@ cdef extern from "numpy/arrayobject.h":
 
     ctypedef void (*PyArray_VectorUnaryFunc)(void *, void *, npy_intp, void *,  void *)
 
+    ctypedef struct PyArray_Descr:
+        pass
+    
     ctypedef class numpy.dtype [object PyArray_Descr]:
         # Use PyDataType_* macros when possible, however there are no macros
         # for accessing some of the fields, so some are defined. Please
@@ -177,15 +180,11 @@ cdef extern from "numpy/arrayobject.h":
     ctypedef class numpy.ndarray [object PyArrayObject]:
         cdef __cythonbufferdefaults__ = {"mode": "strided"}
 
-        cdef:
-            # Only taking a few of the most commonly used and stable fields.
-            # One should use PyArray_* macros instead to access the C fields.
-            char *data
-            int ndim "nd"
-            npy_intp *shape "dimensions"
-            npy_intp *strides
-            dtype descr
-            PyObject* base
+        # Note: The fields are no longer defined, please use accessor
+        # functions. Cython special-cases/hacks the data, ndim, shape
+        # and stride attributes of the ndarray to use accessor
+        # functions for backwards compatability and convenience.
+
 
         # Note: This syntax (function definition in pxd files) is an
         # experimental exception made for __getbuffer__ and __releasebuffer__
@@ -236,7 +235,7 @@ cdef extern from "numpy/arrayobject.h":
 
             cdef int t
             cdef char* f = NULL
-            cdef dtype descr = self.descr
+            cdef dtype descr = get_array_dtype(self)
             cdef list stack
             cdef int offset
 
@@ -376,20 +375,29 @@ cdef extern from "numpy/arrayobject.h":
     bint PyArray_ISWRITEABLE(ndarray m)
     bint PyArray_ISALIGNED(ndarray m)
 
-    int PyArray_NDIM(ndarray)
+    int PyArray_NDIM(ndarray) nogil
     bint PyArray_ISONESEGMENT(ndarray)
     bint PyArray_ISFORTRAN(ndarray)
     int PyArray_FORTRANIF(ndarray)
 
-    void* PyArray_DATA(ndarray)
-    char* PyArray_BYTES(ndarray)
-    npy_intp* PyArray_DIMS(ndarray)
-    npy_intp* PyArray_STRIDES(ndarray)
-    npy_intp PyArray_DIM(ndarray, size_t)
-    npy_intp PyArray_STRIDE(ndarray, size_t)
+    void* PyArray_DATA(ndarray) nogil
+    char* PyArray_BYTES(ndarray) nogil
+    npy_intp* PyArray_DIMS(ndarray) nogil
+    npy_intp* PyArray_STRIDES(ndarray) nogil
+    npy_intp PyArray_DIM(ndarray, size_t) nogil
+    npy_intp PyArray_STRIDE(ndarray, size_t) nogil
+
+    # The two functions below return borrowed references and should
+    # be used with care; often you will want to use get_array_base
+    # or get_array_dtype (define below) instead from Cython.
+    PyObject* PyArray_BASE(ndarray)
+    # Cython API of the function below might change! PyArray_DESCR
+    # actually returns PyArray_Descr* == pointer-version of dtype,
+    # which appears to be difficult to declare properly in Cython;
+    # protect it with trailing underscore for now just to avoid having
+    # user code depend on it without reading this note.
+    PyArray_Descr * PyArray_DESCR_ "PyArray_DESCR"(ndarray)
 
-    # object PyArray_BASE(ndarray) wrong refcount semantics
-    # dtype PyArray_DESCR(ndarray) wrong refcount semantics
     int PyArray_FLAGS(ndarray)
     npy_intp PyArray_ITEMSIZE(ndarray)
     int PyArray_TYPE(ndarray arr)
@@ -961,18 +969,34 @@ cdef extern from "numpy/ufuncobject.h":
     void import_ufunc()
 
 
-cdef inline void set_array_base(ndarray arr, object base):
-     cdef PyObject* baseptr
-     if base is None:
-         baseptr = NULL
-     else:
-         Py_INCREF(base) # important to do this before decref below!
-         baseptr = <PyObject*>base
-     Py_XDECREF(arr.base)
-     arr.base = baseptr
+# The ability to set the base field of an ndarray seems to be
+# deprecated in NumPy 1.7 (no PyArray_SET_BASE seems to be
+# available). Remove this support and see who complains and how their
+# case could be fixed in 1.7...
+#
+#cdef inline void set_array_base(ndarray arr, object base):
+#     cdef PyObject* baseptr
+#     if base is None:
+#         baseptr = NULL
+#     else:
+#         Py_INCREF(base) # important to do this before decref below!
+#         baseptr = <PyObject*>base
+#     Py_XDECREF(arr.base)
+#     arr.base = baseptr
 
 cdef inline object get_array_base(ndarray arr):
-    if arr.base is NULL:
+    cdef PyObject *pobj = PyArray_BASE(arr)
+    if pobj != NULL:
+        obj = <object>pobj
+        Py_INCREF(obj)
+        return obj
+    else:
         return None
+
+cdef inline dtype get_array_dtype(ndarray arr):
+    if PyArray_DESCR_(arr) != NULL:
+        obj = <object>PyArray_DESCR_(arr)
+        Py_INCREF(obj)
+        return obj
     else:
-        return <object>arr.base
+        return None

tests/run/numpy_attributes.pyx

+# tag: numpy
+
+
+import numpy as np
+cimport numpy as np
+
+
+def f():
+    """
+    >>> f()
+    ndim 2
+    data 1
+    shape 3 2
+    shape[1] 2
+    strides 16 8
+    """
+    cdef np.ndarray x = np.ones((3, 2), dtype=np.int64)
+    cdef int i
+    cdef Py_ssize_t j, k
+    cdef char *p
+    # todo: int * p: 23:13: Cannot assign type 'char *' to 'int *'
+
+    with nogil:
+        i = x.ndim
+    print 'ndim', i
+    
+    with nogil:
+        p = x.data
+    print 'data', (<np.int64_t*>p)[0]
+
+    with nogil:
+        j = x.shape[0]
+        k = x.shape[1]
+    print 'shape', j, k
+    # Check that non-typical uses still work
+    cdef np.npy_intp *shape
+    with nogil:
+        shape = x.shape + 1
+    print 'shape[1]', shape[0]
+
+    with nogil:
+        j = x.strides[0]
+        k = x.strides[1]
+    print 'strides', j, k
+