Commit 20a4a188 authored by Dag Sverre Seljebotn's avatar Dag Sverre Seljebotn

Buffer format string checking rewrite

parent 1b176d9a
This diff is collapsed.
...@@ -596,13 +596,16 @@ class CCodeWriter(object): ...@@ -596,13 +596,16 @@ class CCodeWriter(object):
def exit_cfunc_scope(self): def exit_cfunc_scope(self):
self.funcstate = None self.funcstate = None
def putln(self, code = ""): def putln(self, code = "", safe=False):
if self.marker and self.bol: if self.marker and self.bol:
self.emit_marker() self.emit_marker()
if self.emit_linenums and self.last_marker_line != 0: if self.emit_linenums and self.last_marker_line != 0:
self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc)) self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc))
if code: if code:
self.put(code) if safe:
self.put_safe(code)
else:
self.put(code)
self.write("\n"); self.write("\n");
self.bol = 1 self.bol = 1
......
...@@ -143,6 +143,12 @@ class PyrexType(BaseType): ...@@ -143,6 +143,12 @@ class PyrexType(BaseType):
return (self.is_int or self.is_float or self.is_pyobject or return (self.is_int or self.is_float or self.is_pyobject or
self.is_extension_type or self.is_ptr) self.is_extension_type or self.is_ptr)
def struct_nesting_depth(self):
# Returns the number levels of nested structs. This is
# used for constructing a stack for walking the run-time
# type information of the struct.
return 1
class CTypedefType(BaseType): class CTypedefType(BaseType):
# #
# Pseudo-type defined with a ctypedef statement in a # Pseudo-type defined with a ctypedef statement in a
...@@ -1117,6 +1123,10 @@ class CStructOrUnionType(CType): ...@@ -1117,6 +1123,10 @@ class CStructOrUnionType(CType):
fields = self.scope.var_entries fields = self.scope.var_entries
return len(fields) == 2 and fields[0].type.is_float and fields[1].type.is_float return len(fields) == 2 and fields[0].type.is_float and fields[1].type.is_float
def struct_nesting_depth(self):
child_depths = [x.type.struct_nesting_depth()
for x in self.scope.var_entries]
return max(child_depths) + 1
class CEnumType(CType): class CEnumType(CType):
# name string # name string
......
...@@ -274,10 +274,10 @@ cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset ...@@ -274,10 +274,10 @@ cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset
new_byteorder = child.byteorder new_byteorder = child.byteorder
if new_byteorder == '|': new_byteorder = '=' if new_byteorder == '|': new_byteorder = '='
if byteorder[0] != new_byteorder: # if byteorder[0] != new_byteorder:
f[0] = new_byteorder # f[0] = new_byteorder
f += 1 # f += 1
byteorder[0] = new_byteorder # byteorder[0] = new_byteorder
# Output padding bytes # Output padding bytes
while offset[0] < new_offset: while offset[0] < new_offset:
......
...@@ -341,60 +341,6 @@ def explicitly_release_buffer(): ...@@ -341,60 +341,6 @@ def explicitly_release_buffer():
x = None x = None
print "After release" print "After release"
#
# Format strings
#
@testcase
def alignment_string(object[int] buf):
"""
>>> alignment_string(IntMockBuffer(None, [1,2], format="@i"))
2
>>> alignment_string(IntMockBuffer(None, [1,2], format="@i@@"))
2
>>> alignment_string(IntMockBuffer(None, [1,2], format=">i"))
Traceback (most recent call last):
...
ValueError: Buffer acquisition error: Only native byte order, size and alignment supported.
>>> alignment_string(IntMockBuffer(None, [1,2], format="<i"))
Traceback (most recent call last):
...
ValueError: Buffer acquisition error: Only native byte order, size and alignment supported.
>>> alignment_string(IntMockBuffer(None, [1,2], format="=i"))
Traceback (most recent call last):
...
ValueError: Buffer acquisition error: Only native byte order, size and alignment supported.
>>> alignment_string(IntMockBuffer(None, [1,2], format="!i"))
Traceback (most recent call last):
...
ValueError: Buffer acquisition error: Only native byte order, size and alignment supported.
"""
print buf[1]
@testcase
def wrong_string(object[int] buf):
"""
>>> wrong_string(IntMockBuffer(None, [1,2], format="if"))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch (expected end, got float)
>>> wrong_string(IntMockBuffer(None, [1,2], format="$$"))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch (expected int, got unparseable format string)
"""
print buf[1]
@testcase
def int_and_long_are_same():
"""
>>> int_and_long_are_same()
"""
cdef object[int] intarr
cdef object[long] longarr
if sizeof(int) == sizeof(long):
intarr = IntMockBuffer(None, [1,2], format='l')
longarr = IntMockBuffer(None, [1,2])
# #
# Getting items and index bounds checking # Getting items and index bounds checking
# #
...@@ -532,39 +478,6 @@ def no_negative_indices(object[int, negative_indices=False] buf, int idx): ...@@ -532,39 +478,6 @@ def no_negative_indices(object[int, negative_indices=False] buf, int idx):
""" """
return buf[idx] return buf[idx]
#
# Buffer type mismatch examples. Varying the type and access
# method simultaneously, the odds of an interaction is virtually
# zero.
#
@testcase
def fmtst1(buf):
"""
>>> fmtst1(IntMockBuffer("A", range(3)))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch (expected float, got int)
"""
cdef object[float] a = buf
@testcase
def fmtst2(object[int] buf):
"""
>>> fmtst2(FloatMockBuffer("A", range(3)))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch (expected int, got float)
"""
@testcase
def ndim1(object[int, ndim=2] buf):
"""
>>> ndim1(IntMockBuffer("A", range(3)))
Traceback (most recent call last):
...
ValueError: Buffer has wrong number of dimensions (expected 2, got 1)
"""
# #
# Test which flags are passed. # Test which flags are passed.
# #
...@@ -860,8 +773,7 @@ def printbuf_td_cy_int(object[td_cy_int] buf, shape): ...@@ -860,8 +773,7 @@ def printbuf_td_cy_int(object[td_cy_int] buf, shape):
>>> printbuf_td_cy_int(ShortMockBuffer(None, range(3)), (3,)) >>> printbuf_td_cy_int(ShortMockBuffer(None, range(3)), (3,))
Traceback (most recent call last): Traceback (most recent call last):
... ...
ValueError: Buffer dtype mismatch (expected bufaccess.td_cy_int, got short) ValueError: Buffer dtype mismatch, expected 'bufaccess.td_cy_int' but got 'short'
""" """
cdef int i cdef int i
for i in range(shape[0]): for i in range(shape[0]):
...@@ -876,7 +788,7 @@ def printbuf_td_h_short(object[td_h_short] buf, shape): ...@@ -876,7 +788,7 @@ def printbuf_td_h_short(object[td_h_short] buf, shape):
>>> printbuf_td_h_short(IntMockBuffer(None, range(3)), (3,)) >>> printbuf_td_h_short(IntMockBuffer(None, range(3)), (3,))
Traceback (most recent call last): Traceback (most recent call last):
... ...
ValueError: Buffer dtype mismatch (expected bufaccess.td_h_short, got int) ValueError: Buffer dtype mismatch, expected 'bufaccess.td_h_short' but got 'int'
""" """
cdef int i cdef int i
for i in range(shape[0]): for i in range(shape[0]):
...@@ -891,7 +803,7 @@ def printbuf_td_h_cy_short(object[td_h_cy_short] buf, shape): ...@@ -891,7 +803,7 @@ def printbuf_td_h_cy_short(object[td_h_cy_short] buf, shape):
>>> printbuf_td_h_cy_short(IntMockBuffer(None, range(3)), (3,)) >>> printbuf_td_h_cy_short(IntMockBuffer(None, range(3)), (3,))
Traceback (most recent call last): Traceback (most recent call last):
... ...
ValueError: Buffer dtype mismatch (expected bufaccess.td_h_cy_short, got int) ValueError: Buffer dtype mismatch, expected 'bufaccess.td_h_cy_short' but got 'int'
""" """
cdef int i cdef int i
for i in range(shape[0]): for i in range(shape[0]):
...@@ -906,7 +818,7 @@ def printbuf_td_h_ushort(object[td_h_ushort] buf, shape): ...@@ -906,7 +818,7 @@ def printbuf_td_h_ushort(object[td_h_ushort] buf, shape):
>>> printbuf_td_h_ushort(ShortMockBuffer(None, range(3)), (3,)) >>> printbuf_td_h_ushort(ShortMockBuffer(None, range(3)), (3,))
Traceback (most recent call last): Traceback (most recent call last):
... ...
ValueError: Buffer dtype mismatch (expected bufaccess.td_h_ushort, got short) ValueError: Buffer dtype mismatch, expected 'bufaccess.td_h_ushort' but got 'short'
""" """
cdef int i cdef int i
for i in range(shape[0]): for i in range(shape[0]):
...@@ -921,7 +833,7 @@ def printbuf_td_h_double(object[td_h_double] buf, shape): ...@@ -921,7 +833,7 @@ def printbuf_td_h_double(object[td_h_double] buf, shape):
>>> printbuf_td_h_double(FloatMockBuffer(None, [0.25, 1, 3.125]), (3,)) >>> printbuf_td_h_double(FloatMockBuffer(None, [0.25, 1, 3.125]), (3,))
Traceback (most recent call last): Traceback (most recent call last):
... ...
ValueError: Buffer dtype mismatch (expected bufaccess.td_h_double, got float) ValueError: Buffer dtype mismatch, expected 'bufaccess.td_h_double' but got 'float'
""" """
cdef int i cdef int i
for i in range(shape[0]): for i in range(shape[0]):
...@@ -1033,7 +945,7 @@ def buffer_cast_fails(object[char, cast=True] buf): ...@@ -1033,7 +945,7 @@ def buffer_cast_fails(object[char, cast=True] buf):
>>> buffer_cast_fails(IntMockBuffer(None, [0])) >>> buffer_cast_fails(IntMockBuffer(None, [0]))
Traceback (most recent call last): Traceback (most recent call last):
... ...
ValueError: Item size of buffer does not match size of 'char' ValueError: Item size of buffer (4 bytes) does not match size of 'char' (1 byte)
""" """
return buf[0] return buf[0]
...@@ -1366,48 +1278,31 @@ cdef class NestedStructMockBuffer(MockBuffer): ...@@ -1366,48 +1278,31 @@ cdef class NestedStructMockBuffer(MockBuffer):
@testcase @testcase
def basic_struct(object[MyStruct] buf): def basic_struct(object[MyStruct] buf):
""" """
See also buffmt.pyx
>>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)])) >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
1 2 3 4 5 1 2 3 4 5
>>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="bbqii")) >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="bbqii"))
1 2 3 4 5 1 2 3 4 5
>>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="23bqii"))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch (expected long long, got char)
>>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="i"))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch (expected char, got int)
""" """
print buf[0].a, buf[0].b, buf[0].c, buf[0].d, buf[0].e print buf[0].a, buf[0].b, buf[0].c, buf[0].d, buf[0].e
@testcase @testcase
def nested_struct(object[NestedStruct] buf): def nested_struct(object[NestedStruct] buf):
""" """
See also buffmt.pyx
>>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)])) >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
1 2 3 4 5 1 2 3 4 5
>>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i")) >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i"))
1 2 3 4 5 1 2 3 4 5
>>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="iiiii"))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch (expected SmallStruct, got int)
>>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{iii}T{ii}i"))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch (expected end of SmallStruct struct, got int)
""" """
print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z
cdef struct LongComplex: cdef struct LongComplex:
long double real long double real
long double imag long double imag
cdef struct MixedComplex:
long double real
float imag
cdef class LongComplexMockBuffer(MockBuffer): cdef class LongComplexMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1: cdef int write(self, char* buf, object value) except -1:
cdef LongComplex* s cdef LongComplex* s
...@@ -1427,18 +1322,6 @@ def complex_struct_dtype(object[LongComplex] buf): ...@@ -1427,18 +1322,6 @@ def complex_struct_dtype(object[LongComplex] buf):
""" """
print buf[0].real, buf[0].imag print buf[0].real, buf[0].imag
@testcase
def mixed_complex_struct_dtype(object[MixedComplex] buf):
"""
Triggering a specific execution path for this case.
>>> mixed_complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)]))
Traceback (most recent call last):
...
ValueError: Cannot store complex number in 'MixedComplex' as 'long double' differs from 'float' in size.
"""
print buf[0].real, buf[0].imag
@testcase @testcase
def complex_struct_inplace(object[LongComplex] buf): def complex_struct_inplace(object[LongComplex] buf):
""" """
...@@ -1448,7 +1331,7 @@ def complex_struct_inplace(object[LongComplex] buf): ...@@ -1448,7 +1331,7 @@ def complex_struct_inplace(object[LongComplex] buf):
buf[0].real += 1 buf[0].real += 1
buf[0].imag += 2 buf[0].imag += 2
print buf[0].real, buf[0].imag print buf[0].real, buf[0].imag
# #
# Nogil # Nogil
# #
......
# Tests buffer format string parsing.
__test__ = {}
def testcase(func):
__test__[func.__name__] = func.__doc__
return func
cimport stdlib
def little_endian():
cdef unsigned int n = 1
return (<unsigned char*>&n)[0] != 0
if little_endian():
current_endian = '<'
other_endian = '>'
else:
current_endian = '>'
other_endian = '<'
cdef class MockBuffer:
cdef Py_ssize_t zero
cdef Py_ssize_t minusone
cdef object format
cdef object itemsize
def __init__(self, format, itemsize):
self.format = format
self.itemsize = itemsize
self.zero = 0
self.minusone = -1
def __getbuffer__(self, Py_buffer* info, int flags):
info.buf = NULL
info.strides = &self.zero
info.suboffsets = &self.minusone
info.shape = &self.zero
info.ndim = 1
info.format = self.format
info.itemsize = self.itemsize
@testcase
def _int(fmt):
"""
>>> _int("i")
>>> _int("b")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'int' but got 'char'
>>> _int("if")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected end but got 'float'
>>> _int("$$")
Traceback (most recent call last):
...
ValueError: Does not understand character buffer dtype format string ('$')
"""
cdef object[int] buf = MockBuffer(fmt, sizeof(int))
@testcase
def _ulong(fmt):
"""
>>> _ulong("L")
"""
cdef object[unsigned long] buf = MockBuffer(fmt, sizeof(unsigned long))
@testcase
def wrongsize():
"""
>>> wrongsize()
Traceback (most recent call last):
...
ValueError: Item size of buffer (1 byte) does not match size of 'unsigned long' (8 bytes)
"""
cdef object[unsigned long] buf = MockBuffer("L", 1)
@testcase
def _obj(fmt):
"""
>>> _obj("O")
>>> _obj("i")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'Python object' but got 'int'
"""
cdef object[object] buf = MockBuffer(fmt, sizeof(void*))
cdef struct ComplexDouble:
double real
double imag
ctypedef struct Char3Int:
char a
int b
int c
int d
cdef struct CharIntCDouble:
char a
int b
ComplexDouble c
double d
cdef struct UnpackedStruct1:
char a
int b
ComplexDouble c
double c2
Char3Int d
ctypedef struct UnpackedStruct2:
CharIntCDouble a
Char3Int b
ctypedef struct UnpackedStruct3:
CharIntCDouble a
char b
int c, d, e
cdef struct UnpackedStruct4:
char a
int b
ComplexDouble c
double c2
char d
int e, f, g
@testcase
def char3int(fmt):
"""
>>> char3int("ciii")
>>> char3int("c1i1i1i")
>>> char3int("c3i")
>>> char3int("ci2i")
>>> char3int("c@i@2i")
>>> char3int("cii")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'int' but got end in 'Char3Int.d'
"""
obj = MockBuffer(fmt, sizeof(Char3Int))
cdef object[Char3Int, ndim=1] buf = obj
#@testcase
def unpacked_struct(fmt):
"""
Native formats:
>>> unpacked_struct("biZddbiii")
>>> unpacked_struct("@bi3db3i")
>>> unpacked_struct("@biZddbi2i")
>>> unpacked_struct("bidT{biii}")
>>> unpacked_struct("bT{idddb2i}i")
>>> unpacked_struct("bidb3T{i}")
>>> unpacked_struct("T{b}T{T{iZddT{bi}}}2T{T{i}}")
"""
assert (sizeof(UnpackedStruct1) == sizeof(UnpackedStruct2)
== sizeof(UnpackedStruct3) == sizeof(UnpackedStruct4))
obj = MockBuffer(fmt, sizeof(UnpackedStruct1))
cdef object[UnpackedStruct1, ndim=1] buf1 = obj
cdef object[UnpackedStruct2, ndim=1] buf2 = obj
cdef object[UnpackedStruct3, ndim=1] buf3 = obj
cdef object[UnpackedStruct4, ndim=1] buf4 = obj
cdef struct ComplexTest:
ComplexDouble a, b, c
@testcase
def complex_test(fmt):
"""
>>> complex_test("ZdZdZd")
>>> complex_test("3Zd")
>>> complex_test("6d")
>>> complex_test("3T{Zd}")
>>> complex_test("dZdZdd")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'double' but got 'complex double' in 'ComplexDouble.imag'
"""
obj = MockBuffer(fmt, sizeof(ComplexTest))
cdef object[ComplexTest] buf1 = obj
@testcase
def alignment_string(fmt, exc=None):
"""
>>> alignment_string("@i")
>>> alignment_string("@i@@")
>>> alignment_string("%si" % current_endian)
>>> alignment_string("%si" % other_endian, "X-endian buffer not supported on X-endian compiler")
>>> alignment_string("=i")
"""
cdef object[int] buf
try:
buf = MockBuffer(fmt, sizeof(int))
except ValueError, e:
msg = e.message.replace("Big", "X").replace("Little", "X").replace("big", "X").replace("little", "X")
if msg != exc:
print msg
print " is not equal to"
print exc
return
if exc:
print "fail"
@testcase
def int_and_long_are_same():
"""
>>> int_and_long_are_same()
"""
cdef object[int] intarr
cdef object[long] longarr
if sizeof(int) == sizeof(long):
intarr = MockBuffer("l", sizeof(int))
longarr = MockBuffer("i", sizeof(int))
cdef struct MixedComplex:
long double real
float imag
@testcase
def mixed_complex_struct():
"""
Triggering a specific execution path for this case.
>>> mixed_complex_struct()
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'long double' but got 'complex double' in 'MixedComplex.real'
"""
cdef object[MixedComplex] buf = MockBuffer("Zd", sizeof(MixedComplex))
# TODO: empty struct
# TODO: Incomplete structs
...@@ -111,6 +111,7 @@ try: ...@@ -111,6 +111,7 @@ try:
>>> test_dtype('I', inc1_uint) >>> test_dtype('I', inc1_uint)
>>> test_dtype('l', inc1_long) >>> test_dtype('l', inc1_long)
>>> test_dtype('L', inc1_ulong) >>> test_dtype('L', inc1_ulong)
>>> test_dtype('f', inc1_float) >>> test_dtype('f', inc1_float)
>>> test_dtype('d', inc1_double) >>> test_dtype('d', inc1_double)
>>> test_dtype('g', inc1_longdouble) >>> test_dtype('g', inc1_longdouble)
...@@ -144,19 +145,21 @@ try: ...@@ -144,19 +145,21 @@ try:
]))) ])))
Traceback (most recent call last): Traceback (most recent call last):
... ...
ValueError: Buffer dtype mismatch (expected int, got float) ValueError: Buffer dtype mismatch, expected 'int' but got 'float' in 'DoubleInt.y'
>>> test_good_cast() >>> test_good_cast()
True True
>>> test_bad_cast() >>> test_bad_cast()
Traceback (most recent call last): Traceback (most recent call last):
... ...
ValueError: Attempted cast of buffer to datatype of different size. ValueError: Item size of buffer (1 byte) does not match size of 'long' (8 bytes)
""" """
except: except:
__doc__ = u"" __doc__ = u""
def ndarray_str(arr): def ndarray_str(arr):
u""" u"""
Since Py2.3 doctest don't support <BLANKLINE>, manually replace blank lines Since Py2.3 doctest don't support <BLANKLINE>, manually replace blank lines
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment