support for new buffer protocol in Py3

Cython/Compiler/Builtin.py

@@ -2,8 +2,9 @@
 #   Pyrex - Builtin Definitions
 #
 
-from Symtab import BuiltinScope
+from Symtab import BuiltinScope, StructOrUnionScope
 from TypeSlots import Signature
+import PyrexTypes
 
 builtin_function_table = [
     # name,        args,   return,  C API func,           py equiv = "*"
@@ -100,6 +101,21 @@ builtin_types_table = [
                                     ("values","O",   "O", "PyDict_Values")]),
 ]
 
+builtin_structs_table = [
+    ('Py_buffer', 'Py_buffer',
+     [("buf",        PyrexTypes.c_void_ptr_type),
+      ("len",        PyrexTypes.c_py_ssize_t_type),
+      ("readonly",   PyrexTypes.c_bint_type),
+      ("format",     PyrexTypes.c_char_ptr_type),
+      ("ndim",       PyrexTypes.c_int_type),
+      ("shape",      PyrexTypes.c_py_ssize_t_ptr_type),
+      ("strides",    PyrexTypes.c_py_ssize_t_ptr_type),
+      ("suboffsets", PyrexTypes.c_py_ssize_t_ptr_type),
+      ("itemsize",   PyrexTypes.c_py_ssize_t_type),
+      ("internal",   PyrexTypes.c_void_ptr_type),
+      ])
+]
+
 getattr3_utility_code = ["""
 static PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); /*proto*/
 ""","""
@@ -151,9 +167,19 @@ def init_builtin_types():
             sig = Signature(args, ret)
             the_type.scope.declare_cfunction(name, sig.function_type(), None, cname)
 
+def init_builtin_structs():
+    for name, cname, attribute_types in builtin_structs_table:
+        scope = StructOrUnionScope(name)
+        for attribute_name, attribute_type in attribute_types:
+            scope.declare_var(
+                attribute_name, attribute_type, None, attribute_name)
+        builtin_scope.declare_struct_or_union(
+            name, "struct", scope, 1, None, cname = cname)
+
 def init_builtins():
     init_builtin_funcs()
     init_builtin_types()
+    init_builtin_structs()
     global list_type, tuple_type, dict_type
     list_type  = builtin_scope.lookup('list').type
     tuple_type = builtin_scope.lookup('tuple').type

Cython/Compiler/ModuleNode.py

@@ -388,6 +388,30 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define Py_SIZE(ob)   ((PyVarObject*)(ob))->ob_size)")
         code.putln("  #define PyVarObject_HEAD_INIT(type, size) \\")
         code.putln("          PyObject_HEAD_INIT(type) size,")
+        code.putln("")
+        code.putln("  typedef struct {")
+        code.putln("     void *buf;")
+        code.putln("     Py_ssize_t len;")
+        code.putln("     int readonly;")
+        code.putln("     const char *format;")
+        code.putln("     int ndim;")
+        code.putln("     Py_ssize_t *shape;")
+        code.putln("     Py_ssize_t *strides;")
+        code.putln("     Py_ssize_t *suboffsets;")
+        code.putln("     Py_ssize_t itemsize;")
+        code.putln("     void *internal;")
+        code.putln("  } Py_buffer;")
+        code.putln("")
+        code.putln("  #define PyBUF_SIMPLE 0")
+        code.putln("  #define PyBUF_WRITABLE 0x0001")
+        code.putln("  #define PyBUF_LOCK 0x0002")
+        code.putln("  #define PyBUF_FORMAT 0x0004")
+        code.putln("  #define PyBUF_ND 0x0008")
+        code.putln("  #define PyBUF_STRIDES (0x0010 | PyBUF_ND)")
+        code.putln("  #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES)")
+        code.putln("  #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)")
+        code.putln("  #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)")
+        code.putln("  #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)")
         code.putln("#endif")
 
         code.put(builtin_module_name_utility_code[0])

Cython/Compiler/Parsing.py

@@ -2136,6 +2136,7 @@ def p_doc_string(s):
 
 def p_module(s, pxd, full_module_name):
     s.add_type_name("object")
+    s.add_type_name("Py_buffer")
     pos = s.position()
     doc = p_doc_string(s)
     if pxd:

Cython/Compiler/PyrexTypes.py

@@ -1049,12 +1049,16 @@ c_char_array_type =   CCharArrayType(None)
 c_utf8_char_array_type =   CUTF8CharArrayType(None)
 c_char_ptr_type =     CCharPtrType()
 c_char_ptr_ptr_type = CPtrType(c_char_ptr_type)
+c_py_ssize_t_ptr_type =  CPtrType(c_py_ssize_t_type)
 c_int_ptr_type =      CPtrType(c_int_type)
 
 c_returncode_type =   CIntType(2, 1, "T_INT", is_returncode = 1)
 
 c_anon_enum_type =    CAnonEnumType(-1, 1)
 
+# the Py_buffer type is defined in Builtin.py
+c_py_buffer_ptr_type = CPtrType(CStructOrUnionType("Py_buffer", "struct", None, 1, "Py_buffer"))
+
 error_type =    ErrorType()
 
 lowest_float_rank = 6

Cython/Compiler/TypeSlots.py

@@ -26,12 +26,14 @@ class Signature:
     #    'p'  void *
     #    'P'  void **
     #    'i'  int
+    #    'b'  bint
     #    'I'  int *
     #    'l'  long
     #    'Z'  Py_ssize_t
     #    's'  char *
     #    'S'  char **
     #    'r'  int used only to signal exception
+    #    'B'  Py_buffer *
     #    '-'  dummy 'self' argument (not used)
     #    '*'  rest of args passed as generic Python
     #           arg tuple and kw dict (must be last
@@ -50,6 +52,7 @@ class Signature:
         's': PyrexTypes.c_char_ptr_type,
         'S': PyrexTypes.c_char_ptr_ptr_type,
         'r': PyrexTypes.c_returncode_type,
+        'B': PyrexTypes.c_py_buffer_ptr_type,
         # 'T', '-' and '*' are handled otherwise
         # and are not looked up in here
     }
@@ -123,12 +126,14 @@ class SlotDescriptor:
     #  is_initialised_dynamically    Is initialised by code in the module init function
     #  flag                          Py_TPFLAGS_XXX value indicating presence of slot
     #  py3k                          Indicates presence of slot in Python 3
+    #  py2                           Indicates presence of slot in Python 2
 
-    def __init__(self, slot_name, dynamic = 0, flag = None, py3k = True):
+    def __init__(self, slot_name, dynamic = 0, flag = None, py3k = True, py2 = True):
         self.slot_name = slot_name
         self.is_initialised_dynamically = dynamic
         self.flag = flag
         self.py3k = py3k
+        self.py2  = py2
 
     def generate(self, scope, code):
         if self.is_initialised_dynamically:
@@ -137,15 +142,18 @@ class SlotDescriptor:
             value = self.slot_code(scope)
         flag = self.flag
         py3k = self.py3k
+        py2  = self.py2
         if not py3k:
             code.putln("#if PY_MAJOR_VERSION < 3")
+        elif not py2:
+            code.putln("#if PY_MAJOR_VERSION >= 3")
         if flag:
             code.putln("#if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & %s)" % flag)
         code.putln("%s, /*%s*/" % (value, self.slot_name))
-        if not py3k:
-            code.putln("#endif")
         if flag:
             code.putln("#endif")
+        if not py3k or not py2:
+            code.putln("#endif")
 
     # Some C implementations have trouble statically 
     # initialising a global with a pointer to an extern 
@@ -191,8 +199,8 @@ class MethodSlot(SlotDescriptor):
     #  method_name  string           The __xxx__ name of the method
     #  default      string or None   Default value of the slot
     
-    def __init__(self, signature, slot_name, method_name, default = None, flag = None, py3k=True):
-        SlotDescriptor.__init__(self, slot_name, flag = flag, py3k = py3k)
+    def __init__(self, signature, slot_name, method_name, default = None, flag = None, py3k=True, py2=True):
+        SlotDescriptor.__init__(self, slot_name, flag = flag, py3k = py3k, py2=py2)
         self.signature = signature
         self.slot_name = slot_name
         self.method_name = method_name
@@ -213,8 +221,8 @@ class InternalMethodSlot(SlotDescriptor):
     #
     #  slot_name    string           Member name of the slot in the type object
 
-    def __init__(self, slot_name):
-        SlotDescriptor.__init__(self, slot_name)
+    def __init__(self, slot_name, py2 = True):
+        SlotDescriptor.__init__(self, slot_name, py2 = py2)
 
     def slot_code(self, scope):
         return scope.mangle_internal(self.slot_name)
@@ -272,8 +280,8 @@ class SyntheticSlot(InternalMethodSlot):
     #  alternative default value will be placed in the type
     #  slot.
     
-    def __init__(self, slot_name, user_methods, default_value):
-        InternalMethodSlot.__init__(self, slot_name)
+    def __init__(self, slot_name, user_methods, default_value, py2 = True):
+        InternalMethodSlot.__init__(self, slot_name, py2 = py2)
         self.user_methods = user_methods
         self.default_value = default_value
     
@@ -504,6 +512,10 @@ initproc = Signature("T*", 'r')            # typedef int (*initproc)(PyObject *,
                                            # typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
                                            # typedef PyObject *(*allocfunc)(struct _typeobject *, int);
 
+getbufferproc = Signature("TBi", "r")      # typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
+releasebufferproc = Signature("TB", "v")   # typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
+
+
 #------------------------------------------------------------------------------------------
 #
 #  Signatures for accessor methods of properties.
@@ -596,6 +608,9 @@ PyBufferProcs = (
     MethodSlot(getwritebufferproc, "bf_getwritebuffer", "__getwritebuffer__", py3k = False),
     MethodSlot(getsegcountproc, "bf_getsegcount", "__getsegcount__", py3k = False),
     MethodSlot(getcharbufferproc, "bf_getcharbuffer", "__getcharbuffer__", py3k = False),
+
+    MethodSlot(getbufferproc, "bf_getbuffer", "__getbuffer__", flag = "Py_TPFLAGS_HAVE_NEWBUFFER"),
+    MethodSlot(releasebufferproc, "bf_releasebuffer", "__releasebuffer__", flag = "Py_TPFLAGS_HAVE_NEWBUFFER"),
 )
 
 #------------------------------------------------------------------------------------------

tests/run/buffer.pyx

+__doc__ = u"""
+>>> b1 = TestBuffer()
+>>> b2 = TestBufferRelease()
+"""
+
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ += u"""
+>>> ms = memoryview(s)
+>>> ms.tobytes()
+bytearray(b'abcdefg')
+
+>>> m1 = memoryview(b1)
+>>> m1.tobytes()
+locking!
+bytearray(b'abcdefg')
+
+>>> m2 = memoryview(b2)
+>>> m2.tobytes()
+locking!
+unlocking!
+bytearray(b'abcdefg')
+
+>>> del m1
+>>> del m2
+releasing!
+"""
+
+s = "abcdefg"
+
+cdef class TestBuffer:
+    def __getbuffer__(self, Py_buffer* buffer, int flags):
+        if buffer is NULL:
+            print u"locking!"
+            return
+        buffer.buf = <char*>s
+        buffer.len = len(s)
+        buffer.readonly = 0
+        buffer.format = "B"
+        buffer.ndim = 0
+        buffer.shape = NULL
+        buffer.strides = NULL
+        buffer.suboffsets = NULL
+        buffer.itemsize = 1
+        buffer.internal = NULL
+
+cdef class TestBufferRelease(TestBuffer):
+    def __releasebuffer__(self, Py_buffer* buffer):
+        if buffer is NULL:
+            print u"unlocking!"
+        else:
+            print u"releasing!"