Buffer format string checking rewrite

Cython/Compiler/Code.py

@@ -596,12 +596,15 @@ class CCodeWriter(object):
     def exit_cfunc_scope(self):
         self.funcstate = None
 
-    def putln(self, code = ""):
+    def putln(self, code = "", safe=False):
         if self.marker and self.bol:
             self.emit_marker()
         if self.emit_linenums and self.last_marker_line != 0:
             self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc))
         if code:
+            if safe:
+                self.put_safe(code)
+            else:
                 self.put(code)
         self.write("\n");
         self.bol = 1

Cython/Compiler/PyrexTypes.py

@@ -143,6 +143,12 @@ class PyrexType(BaseType):
         return (self.is_int or self.is_float or self.is_pyobject or
                 self.is_extension_type or self.is_ptr)
 
+    def struct_nesting_depth(self):
+        # Returns the number levels of nested structs. This is
+        # used for constructing a stack for walking the run-time
+        # type information of the struct.
+        return 1
+
 class CTypedefType(BaseType):
     #
     #  Pseudo-type defined with a ctypedef statement in a
@@ -1117,6 +1123,10 @@ class CStructOrUnionType(CType):
         fields = self.scope.var_entries
         return len(fields) == 2 and fields[0].type.is_float and fields[1].type.is_float
 
+    def struct_nesting_depth(self):
+        child_depths = [x.type.struct_nesting_depth()
+                        for x in self.scope.var_entries]
+        return max(child_depths) + 1
 
 class CEnumType(CType):
     #  name           string

Cython/Includes/numpy.pxd

@@ -274,10 +274,10 @@ cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset
 
         new_byteorder = child.byteorder
         if new_byteorder == '|': new_byteorder = '='
-        if byteorder[0] != new_byteorder:
-            f[0] = new_byteorder
-            f += 1
-            byteorder[0] = new_byteorder
+#        if byteorder[0] != new_byteorder:
+#            f[0] = new_byteorder
+#            f += 1
+#            byteorder[0] = new_byteorder
 
         # Output padding bytes
         while offset[0] < new_offset:

tests/run/bufaccess.pyx

@@ -341,60 +341,6 @@ def explicitly_release_buffer():
     x = None
     print "After release"
 
-#
-# Format strings
-#
-@testcase
-def alignment_string(object[int] buf):
-    """
-    >>> alignment_string(IntMockBuffer(None, [1,2], format="@i"))
-    2
-    >>> alignment_string(IntMockBuffer(None, [1,2], format="@i@@"))
-    2
-    >>> alignment_string(IntMockBuffer(None, [1,2], format=">i"))
-    Traceback (most recent call last):
-        ...    
-    ValueError: Buffer acquisition error: Only native byte order, size and alignment supported.
-    >>> alignment_string(IntMockBuffer(None, [1,2], format="<i"))
-    Traceback (most recent call last):
-        ...    
-    ValueError: Buffer acquisition error: Only native byte order, size and alignment supported.
-    >>> alignment_string(IntMockBuffer(None, [1,2], format="=i"))
-    Traceback (most recent call last):
-        ...    
-    ValueError: Buffer acquisition error: Only native byte order, size and alignment supported.
-    >>> alignment_string(IntMockBuffer(None, [1,2], format="!i"))
-    Traceback (most recent call last):
-        ...    
-    ValueError: Buffer acquisition error: Only native byte order, size and alignment supported.
-    """ 
-    print buf[1]
-
-@testcase
-def wrong_string(object[int] buf):
-    """
-    >>> wrong_string(IntMockBuffer(None, [1,2], format="if"))
-    Traceback (most recent call last):
-        ...
-    ValueError: Buffer dtype mismatch (expected end, got float)
-    >>> wrong_string(IntMockBuffer(None, [1,2], format="$$"))
-    Traceback (most recent call last):
-        ...
-    ValueError: Buffer dtype mismatch (expected int, got unparseable format string)
-    """
-    print buf[1]
-
-@testcase
-def int_and_long_are_same():
-    """
-    >>> int_and_long_are_same()
-    """
-    cdef object[int] intarr
-    cdef object[long] longarr
-    if sizeof(int) == sizeof(long):
-        intarr = IntMockBuffer(None, [1,2], format='l')
-        longarr = IntMockBuffer(None, [1,2])
-
 #
 # Getting items and index bounds checking
 # 
@@ -532,39 +478,6 @@ def no_negative_indices(object[int, negative_indices=False] buf, int idx):
     """
     return buf[idx]
 
-#
-# Buffer type mismatch examples. Varying the type and access
-# method simultaneously, the odds of an interaction is virtually
-# zero.
-#
-@testcase
-def fmtst1(buf):
-    """
-    >>> fmtst1(IntMockBuffer("A", range(3)))
-    Traceback (most recent call last):
-        ...
-    ValueError: Buffer dtype mismatch (expected float, got int)
-    """
-    cdef object[float] a = buf
-
-@testcase
-def fmtst2(object[int] buf):
-    """
-    >>> fmtst2(FloatMockBuffer("A", range(3)))
-    Traceback (most recent call last):
-        ...
-    ValueError: Buffer dtype mismatch (expected int, got float)
-    """
-
-@testcase
-def ndim1(object[int, ndim=2] buf):
-    """
-    >>> ndim1(IntMockBuffer("A", range(3)))
-    Traceback (most recent call last):
-        ...
-    ValueError: Buffer has wrong number of dimensions (expected 2, got 1)
-    """
-
 #
 # Test which flags are passed.
 #
@@ -860,8 +773,7 @@ def printbuf_td_cy_int(object[td_cy_int] buf, shape):
     >>> printbuf_td_cy_int(ShortMockBuffer(None, range(3)), (3,))
     Traceback (most recent call last):
        ...
-    ValueError: Buffer dtype mismatch (expected bufaccess.td_cy_int, got short)
-    
+    ValueError: Buffer dtype mismatch, expected 'bufaccess.td_cy_int' but got 'short'
     """
     cdef int i
     for i in range(shape[0]):
@@ -876,7 +788,7 @@ def printbuf_td_h_short(object[td_h_short] buf, shape):
     >>> printbuf_td_h_short(IntMockBuffer(None, range(3)), (3,))
     Traceback (most recent call last):
        ...
-    ValueError: Buffer dtype mismatch (expected bufaccess.td_h_short, got int)
+    ValueError: Buffer dtype mismatch, expected 'bufaccess.td_h_short' but got 'int'
     """    
     cdef int i
     for i in range(shape[0]):
@@ -891,7 +803,7 @@ def printbuf_td_h_cy_short(object[td_h_cy_short] buf, shape):
     >>> printbuf_td_h_cy_short(IntMockBuffer(None, range(3)), (3,))
     Traceback (most recent call last):
        ...
-    ValueError: Buffer dtype mismatch (expected bufaccess.td_h_cy_short, got int)
+    ValueError: Buffer dtype mismatch, expected 'bufaccess.td_h_cy_short' but got 'int'
     """
     cdef int i
     for i in range(shape[0]):
@@ -906,7 +818,7 @@ def printbuf_td_h_ushort(object[td_h_ushort] buf, shape):
     >>> printbuf_td_h_ushort(ShortMockBuffer(None, range(3)), (3,))
     Traceback (most recent call last):
        ...
-    ValueError: Buffer dtype mismatch (expected bufaccess.td_h_ushort, got short)
+    ValueError: Buffer dtype mismatch, expected 'bufaccess.td_h_ushort' but got 'short'
     """
     cdef int i
     for i in range(shape[0]):
@@ -921,7 +833,7 @@ def printbuf_td_h_double(object[td_h_double] buf, shape):
     >>> printbuf_td_h_double(FloatMockBuffer(None, [0.25, 1, 3.125]), (3,))
     Traceback (most recent call last):
        ...
-    ValueError: Buffer dtype mismatch (expected bufaccess.td_h_double, got float)
+    ValueError: Buffer dtype mismatch, expected 'bufaccess.td_h_double' but got 'float'
     """
     cdef int i
     for i in range(shape[0]):
@@ -1033,7 +945,7 @@ def buffer_cast_fails(object[char, cast=True] buf):
     >>> buffer_cast_fails(IntMockBuffer(None, [0]))
     Traceback (most recent call last):
         ...
-    ValueError: Item size of buffer does not match size of 'char'
+    ValueError: Item size of buffer (4 bytes) does not match size of 'char' (1 byte)
     """
     return buf[0]
 
@@ -1366,48 +1278,31 @@ cdef class NestedStructMockBuffer(MockBuffer):
 @testcase
 def basic_struct(object[MyStruct] buf):
     """
+    See also buffmt.pyx
+    
     >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
     1 2 3 4 5
     >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="bbqii"))
     1 2 3 4 5
-    >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="23bqii"))
-    Traceback (most recent call last):
-        ...
-    ValueError: Buffer dtype mismatch (expected long long, got char)
-    >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="i"))
-    Traceback (most recent call last):
-        ...
-    ValueError: Buffer dtype mismatch (expected char, got int)
     """
     print buf[0].a, buf[0].b, buf[0].c, buf[0].d, buf[0].e
 
 @testcase
 def nested_struct(object[NestedStruct] buf):
     """
+    See also buffmt.pyx
+    
     >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
     1 2 3 4 5
     >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i"))
     1 2 3 4 5
-    >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="iiiii"))
-    Traceback (most recent call last):
-        ...
-    ValueError: Buffer dtype mismatch (expected SmallStruct, got int)
-    >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{iii}T{ii}i"))
-    Traceback (most recent call last):
-        ...
-    ValueError: Buffer dtype mismatch (expected end of SmallStruct struct, got int)
     """
     print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z
 
-
 cdef struct LongComplex:
     long double real
     long double imag
 
-cdef struct MixedComplex:
-    long double real
-    float imag
-
 cdef class LongComplexMockBuffer(MockBuffer):
     cdef int write(self, char* buf, object value) except -1:
         cdef LongComplex* s
@@ -1427,18 +1322,6 @@ def complex_struct_dtype(object[LongComplex] buf):
     """
     print buf[0].real, buf[0].imag
 
-@testcase
-def mixed_complex_struct_dtype(object[MixedComplex] buf):
-    """
-    Triggering a specific execution path for this case.
- 
-    >>> mixed_complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)]))
-    Traceback (most recent call last):
-        ...
-    ValueError: Cannot store complex number in 'MixedComplex' as 'long double' differs from 'float' in size.
-    """
-    print buf[0].real, buf[0].imag
-
 @testcase
 def complex_struct_inplace(object[LongComplex] buf):
     """

tests/run/buffmt.pyx

+# Tests buffer format string parsing.
+
+__test__ = {}
+def testcase(func):
+    __test__[func.__name__] = func.__doc__
+    return func
+
+cimport stdlib
+
+def little_endian():
+    cdef unsigned int n = 1
+    return (<unsigned char*>&n)[0] != 0
+
+if little_endian():
+    current_endian = '<'
+    other_endian = '>'
+else:
+    current_endian = '>'
+    other_endian = '<'
+ 
+cdef class MockBuffer:
+    cdef Py_ssize_t zero
+    cdef Py_ssize_t minusone
+    cdef object format
+    cdef object itemsize
+    
+    def __init__(self, format, itemsize):
+        self.format = format
+        self.itemsize = itemsize
+        self.zero = 0
+        self.minusone = -1
+    
+    def __getbuffer__(self, Py_buffer* info, int flags):
+        info.buf = NULL
+        info.strides = &self.zero
+        info.suboffsets = &self.minusone
+        info.shape = &self.zero
+        info.ndim = 1
+        info.format = self.format
+        info.itemsize = self.itemsize
+
+@testcase
+def _int(fmt):
+    """
+    >>> _int("i")
+
+    >>> _int("b")
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'int' but got 'char'
+    
+    >>> _int("if")
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected end but got 'float'
+
+    >>> _int("$$")
+    Traceback (most recent call last):
+       ...
+    ValueError: Does not understand character buffer dtype format string ('$')
+    """
+    cdef object[int] buf = MockBuffer(fmt, sizeof(int))
+
+@testcase
+def _ulong(fmt):
+    """
+    >>> _ulong("L")
+    """
+    cdef object[unsigned long] buf = MockBuffer(fmt, sizeof(unsigned long))
+
+@testcase
+def wrongsize():
+    """
+    >>> wrongsize()
+    Traceback (most recent call last):
+       ...
+    ValueError: Item size of buffer (1 byte) does not match size of 'unsigned long' (8 bytes)
+
+    """    
+    cdef object[unsigned long] buf = MockBuffer("L", 1)
+
+@testcase
+def _obj(fmt):
+    """
+    >>> _obj("O")
+    >>> _obj("i")
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'Python object' but got 'int'
+    """
+    cdef object[object] buf = MockBuffer(fmt, sizeof(void*))
+
+
+cdef struct ComplexDouble:
+    double real
+    double imag
+
+ctypedef struct Char3Int:
+    char a
+    int b
+    int c
+    int d
+
+cdef struct CharIntCDouble:
+    char a
+    int b
+    ComplexDouble c
+    double d
+
+cdef struct UnpackedStruct1:
+    char a
+    int b
+    ComplexDouble c
+    double c2
+    Char3Int d
+
+ctypedef struct UnpackedStruct2:
+    CharIntCDouble a
+    Char3Int b
+
+ctypedef struct UnpackedStruct3:
+    CharIntCDouble a
+    char b
+    int c, d, e
+
+cdef struct UnpackedStruct4:
+    char a
+    int b
+    ComplexDouble c
+    double c2
+    char d
+    int e, f, g
+
+@testcase
+def char3int(fmt):
+    """
+    >>> char3int("ciii")
+    >>> char3int("c1i1i1i")    
+    >>> char3int("c3i")
+    >>> char3int("ci2i")
+    >>> char3int("c@i@2i")
+
+    >>> char3int("cii")
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'int' but got end in 'Char3Int.d'
+    """
+    obj = MockBuffer(fmt, sizeof(Char3Int))
+    cdef object[Char3Int, ndim=1] buf = obj
+
+#@testcase
+def unpacked_struct(fmt):
+    """
+    Native formats:
+    >>> unpacked_struct("biZddbiii")
+    >>> unpacked_struct("@bi3db3i")
+    >>> unpacked_struct("@biZddbi2i")
+    >>> unpacked_struct("bidT{biii}")
+    >>> unpacked_struct("bT{idddb2i}i")
+    >>> unpacked_struct("bidb3T{i}")
+    >>> unpacked_struct("T{b}T{T{iZddT{bi}}}2T{T{i}}")
+    """
+
+    assert (sizeof(UnpackedStruct1) == sizeof(UnpackedStruct2)
+            == sizeof(UnpackedStruct3) == sizeof(UnpackedStruct4))
+    obj = MockBuffer(fmt, sizeof(UnpackedStruct1))
+    cdef object[UnpackedStruct1, ndim=1] buf1 = obj
+    cdef object[UnpackedStruct2, ndim=1] buf2 = obj
+    cdef object[UnpackedStruct3, ndim=1] buf3 = obj
+    cdef object[UnpackedStruct4, ndim=1] buf4 = obj
+
+cdef struct ComplexTest:
+    ComplexDouble a, b, c
+
+@testcase
+def complex_test(fmt):
+    """
+    >>> complex_test("ZdZdZd")
+    >>> complex_test("3Zd")
+    >>> complex_test("6d")
+    >>> complex_test("3T{Zd}")
+    
+    >>> complex_test("dZdZdd")
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'double' but got 'complex double' in 'ComplexDouble.imag'
+    
+    """
+    obj = MockBuffer(fmt, sizeof(ComplexTest))
+    cdef object[ComplexTest] buf1 = obj
+    
+
+@testcase
+def alignment_string(fmt, exc=None):
+    """
+    >>> alignment_string("@i")
+    >>> alignment_string("@i@@")
+    >>> alignment_string("%si" % current_endian)
+    >>> alignment_string("%si" % other_endian, "X-endian buffer not supported on X-endian compiler")
+    >>> alignment_string("=i")
+    """
+    cdef object[int] buf
+    try:
+        buf = MockBuffer(fmt, sizeof(int))
+    except ValueError, e:
+        msg = e.message.replace("Big", "X").replace("Little", "X").replace("big", "X").replace("little", "X")
+        if msg != exc:
+            print msg
+            print "  is not equal to"
+            print exc
+        return
+    if exc:
+        print "fail"
+
+
+@testcase
+def int_and_long_are_same():
+    """
+    >>> int_and_long_are_same()
+    """
+    cdef object[int] intarr
+    cdef object[long] longarr
+    if sizeof(int) == sizeof(long):
+        intarr = MockBuffer("l", sizeof(int))
+        longarr = MockBuffer("i", sizeof(int))
+
+cdef struct MixedComplex:
+    long double real
+    float imag
+
+@testcase
+def mixed_complex_struct():
+    """
+    Triggering a specific execution path for this case.
+ 
+    >>> mixed_complex_struct()
+    Traceback (most recent call last):
+        ...
+    ValueError: Buffer dtype mismatch, expected 'long double' but got 'complex double' in 'MixedComplex.real'
+    """
+    cdef object[MixedComplex] buf = MockBuffer("Zd", sizeof(MixedComplex))
+
+ 
+# TODO: empty struct
+# TODO: Incomplete structs

tests/run/numpy_test.pyx

@@ -111,6 +111,7 @@ try:
     >>> test_dtype('I', inc1_uint)
     >>> test_dtype('l', inc1_long)
     >>> test_dtype('L', inc1_ulong)
+    
     >>> test_dtype('f', inc1_float)
     >>> test_dtype('d', inc1_double)
     >>> test_dtype('g', inc1_longdouble)
@@ -144,19 +145,21 @@ try:
         ])))
     Traceback (most recent call last):
         ...
-    ValueError: Buffer dtype mismatch (expected int, got float)
+    ValueError: Buffer dtype mismatch, expected 'int' but got 'float' in 'DoubleInt.y'
+
 
     >>> test_good_cast()
     True
     >>> test_bad_cast()
     Traceback (most recent call last):
         ...
-    ValueError: Attempted cast of buffer to datatype of different size.
+    ValueError: Item size of buffer (1 byte) does not match size of 'long' (8 bytes)
     
 """
 except:
     __doc__ = u""
 
+
 def ndarray_str(arr):
     u"""
     Since Py2.3 doctest don't support <BLANKLINE>, manually replace blank lines