Commit 4efa79b6 authored by Dag Sverre Seljebotn's avatar Dag Sverre Seljebotn

Merge branch 'bug/3118-align'

parents 269fb2b8 fcd59a76
...@@ -747,7 +747,8 @@ typedef struct { ...@@ -747,7 +747,8 @@ typedef struct {
int new_count, enc_count; int new_count, enc_count;
int is_complex; int is_complex;
char enc_type; char enc_type;
char packmode; char new_packmode;
char enc_packmode;
} __Pyx_BufFmt_Context; } __Pyx_BufFmt_Context;
static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx,
...@@ -762,7 +763,8 @@ static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, ...@@ -762,7 +763,8 @@ static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx,
ctx->head->field = &ctx->root; ctx->head->field = &ctx->root;
ctx->fmt_offset = 0; ctx->fmt_offset = 0;
ctx->head->parent_offset = 0; ctx->head->parent_offset = 0;
ctx->packmode = '@'; ctx->new_packmode = '@';
ctx->enc_packmode = '@';
ctx->new_count = 1; ctx->new_count = 1;
ctx->enc_count = 0; ctx->enc_count = 0;
ctx->enc_type = 0; ctx->enc_type = 0;
...@@ -936,12 +938,12 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) { ...@@ -936,12 +938,12 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
__Pyx_StructField* field = ctx->head->field; __Pyx_StructField* field = ctx->head->field;
__Pyx_TypeInfo* type = field->type; __Pyx_TypeInfo* type = field->type;
if (ctx->packmode == '@' || ctx->packmode == '^') { if (ctx->enc_packmode == '@' || ctx->enc_packmode == '^') {
size = __Pyx_BufFmt_TypeCharToNativeSize(ctx->enc_type, ctx->is_complex); size = __Pyx_BufFmt_TypeCharToNativeSize(ctx->enc_type, ctx->is_complex);
} else { } else {
size = __Pyx_BufFmt_TypeCharToStandardSize(ctx->enc_type, ctx->is_complex); size = __Pyx_BufFmt_TypeCharToStandardSize(ctx->enc_type, ctx->is_complex);
} }
if (ctx->packmode == '@') { if (ctx->enc_packmode == '@') {
int align_at = __Pyx_BufFmt_TypeCharToAlignment(ctx->enc_type, ctx->is_complex); int align_at = __Pyx_BufFmt_TypeCharToAlignment(ctx->enc_type, ctx->is_complex);
int align_mod_offset; int align_mod_offset;
if (align_at == 0) return -1; if (align_at == 0) return -1;
...@@ -1008,14 +1010,6 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) { ...@@ -1008,14 +1010,6 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
return 0; return 0;
} }
static int __Pyx_BufFmt_FirstPack(__Pyx_BufFmt_Context* ctx) {
if (ctx->enc_type != 0 || ctx->packmode != '@') {
PyErr_SetString(PyExc_ValueError, "Buffer packing mode currently only allowed at beginning of format string (this is a defect)");
return -1;
}
return 0;
}
static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) { static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) {
int got_Z = 0; int got_Z = 0;
while (1) { while (1) {
...@@ -1041,8 +1035,7 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha ...@@ -1041,8 +1035,7 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
PyErr_SetString(PyExc_ValueError, "Little-endian buffer not supported on big-endian compiler"); PyErr_SetString(PyExc_ValueError, "Little-endian buffer not supported on big-endian compiler");
return NULL; return NULL;
} }
if (__Pyx_BufFmt_FirstPack(ctx) == -1) return NULL; ctx->new_packmode = '=';
ctx->packmode = '=';
++ts; ++ts;
break; break;
case '>': case '>':
...@@ -1051,15 +1044,13 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha ...@@ -1051,15 +1044,13 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
PyErr_SetString(PyExc_ValueError, "Big-endian buffer not supported on little-endian compiler"); PyErr_SetString(PyExc_ValueError, "Big-endian buffer not supported on little-endian compiler");
return NULL; return NULL;
} }
if (__Pyx_BufFmt_FirstPack(ctx) == -1) return NULL; ctx->new_packmode = '=';
ctx->packmode = '=';
++ts; ++ts;
break; break;
case '=': case '=':
case '@': case '@':
case '^': case '^':
if (__Pyx_BufFmt_FirstPack(ctx) == -1) return NULL; ctx->new_packmode = *ts++;
ctx->packmode = *ts++;
break; break;
case 'T': /* substruct */ case 'T': /* substruct */
{ {
...@@ -1090,6 +1081,7 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha ...@@ -1090,6 +1081,7 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
ctx->new_count = 1; ctx->new_count = 1;
ctx->enc_count = 0; ctx->enc_count = 0;
ctx->enc_type = 0; ctx->enc_type = 0;
ctx->enc_packmode = ctx->new_packmode;
++ts; ++ts;
break; break;
case 'Z': case 'Z':
...@@ -1103,13 +1095,15 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha ...@@ -1103,13 +1095,15 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
case 'l': case 'L': case 'q': case 'Q': case 'l': case 'L': case 'q': case 'Q':
case 'f': case 'd': case 'g': case 'f': case 'd': case 'g':
case 'O': case 'O':
if (ctx->enc_type == *ts && got_Z == ctx->is_complex) { if (ctx->enc_type == *ts && got_Z == ctx->is_complex &&
ctx->enc_packmode == ctx->new_packmode) {
/* Continue pooling same type */ /* Continue pooling same type */
ctx->enc_count += ctx->new_count; ctx->enc_count += ctx->new_count;
} else { } else {
/* New type */ /* New type */
if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL;
ctx->enc_count = ctx->new_count; ctx->enc_count = ctx->new_count;
ctx->enc_packmode = ctx->new_packmode;
ctx->enc_type = *ts; ctx->enc_type = *ts;
ctx->is_complex = got_Z; ctx->is_complex = got_Z;
} }
......
...@@ -1281,6 +1281,16 @@ cdef struct NestedStruct: ...@@ -1281,6 +1281,16 @@ cdef struct NestedStruct:
SmallStruct y SmallStruct y
int z int z
cdef packed struct PackedStruct:
char a
int b
cdef struct NestedPackedStruct:
char a
int b
PackedStruct sub
int c
cdef class MyStructMockBuffer(MockBuffer): cdef class MyStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1: cdef int write(self, char* buf, object value) except -1:
cdef MyStruct* s cdef MyStruct* s
...@@ -1301,6 +1311,26 @@ cdef class NestedStructMockBuffer(MockBuffer): ...@@ -1301,6 +1311,26 @@ cdef class NestedStructMockBuffer(MockBuffer):
cdef get_itemsize(self): return sizeof(NestedStruct) cdef get_itemsize(self): return sizeof(NestedStruct)
cdef get_default_format(self): return b"2T{ii}i" cdef get_default_format(self): return b"2T{ii}i"
cdef class PackedStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef PackedStruct* s
s = <PackedStruct*>buf;
s.a, s.b = value
return 0
cdef get_itemsize(self): return sizeof(PackedStruct)
cdef get_default_format(self): return b"^ci"
cdef class NestedPackedStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef NestedPackedStruct* s
s = <NestedPackedStruct*>buf;
s.a, s.b, s.sub.a, s.sub.b, s.c = value
return 0
cdef get_itemsize(self): return sizeof(NestedPackedStruct)
cdef get_default_format(self): return b"ci^ci@i"
@testcase @testcase
def basic_struct(object[MyStruct] buf): def basic_struct(object[MyStruct] buf):
""" """
...@@ -1325,6 +1355,35 @@ def nested_struct(object[NestedStruct] buf): ...@@ -1325,6 +1355,35 @@ def nested_struct(object[NestedStruct] buf):
""" """
print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z
@testcase
def packed_struct(object[PackedStruct] buf):
"""
See also buffmt.pyx
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)]))
1 2
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c^i}"))
1 2
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c=i}"))
1 2
"""
print buf[0].a, buf[0].b
@testcase
def nested_packed_struct(object[NestedPackedStruct] buf):
"""
See also buffmt.pyx
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
1 2 3 4 5
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ci^ci@i"))
1 2 3 4 5
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i"))
1 2 3 4 5
"""
print buf[0].a, buf[0].b, buf[0].sub.a, buf[0].sub.b, buf[0].c
cdef struct LongComplex: cdef struct LongComplex:
long double real long double real
long double imag long double imag
......
...@@ -155,7 +155,7 @@ def char3int(fmt): ...@@ -155,7 +155,7 @@ def char3int(fmt):
>>> char3int("c3i") >>> char3int("c3i")
>>> char3int("ci2i") >>> char3int("ci2i")
#TODO > char3int("c@i@2i") >>> char3int("c@i@2i")
Extra pad bytes (assuming int size is 4 or more) Extra pad bytes (assuming int size is 4 or more)
>>> char3int("cxiii") >>> char3int("cxiii")
...@@ -169,7 +169,7 @@ def char3int(fmt): ...@@ -169,7 +169,7 @@ def char3int(fmt):
... ...
ValueError: Buffer dtype mismatch; next field is at offset 1 but 4 expected ValueError: Buffer dtype mismatch; next field is at offset 1 but 4 expected
#TODO char3int("=cxxx@iii") >>> char3int("=cxxx@iii")
Error: Error:
>>> char3int("cii") >>> char3int("cii")
...@@ -277,11 +277,25 @@ cdef packed struct PackedSubStruct: ...@@ -277,11 +277,25 @@ cdef packed struct PackedSubStruct:
char x char x
int y int y
cdef struct UnpackedSubStruct:
char x
int y
cdef packed struct PackedStruct: cdef packed struct PackedStruct:
char a char a
int b int b
PackedSubStruct sub PackedSubStruct sub
cdef struct PartiallyPackedStruct:
char a
int b
PackedSubStruct sub
cdef packed struct PartiallyPackedStruct2:
char a
UnpackedSubStruct sub
char b
int c
@testcase @testcase
def packed_struct(fmt): def packed_struct(fmt):
...@@ -291,12 +305,13 @@ def packed_struct(fmt): ...@@ -291,12 +305,13 @@ def packed_struct(fmt):
>>> packed_struct("^cici") >>> packed_struct("^cici")
>>> packed_struct("=cibi") >>> packed_struct("=cibi")
However aligned access won't work:
>>> packed_struct("^c@i^ci") >>> packed_struct("^c@i^ci")
Traceback (most recent call last): Traceback (most recent call last):
... ...
ValueError: Buffer packing mode currently only allowed at beginning of format string (this is a defect) ValueError: Buffer dtype mismatch; next field is at offset 4 but 1 expected
However aligned access won't work:
>>> packed_struct("@cici") >>> packed_struct("@cici")
Traceback (most recent call last): Traceback (most recent call last):
... ...
...@@ -305,6 +320,63 @@ def packed_struct(fmt): ...@@ -305,6 +320,63 @@ def packed_struct(fmt):
""" """
cdef object[PackedStruct] buf = MockBuffer(fmt, sizeof(PackedStruct)) cdef object[PackedStruct] buf = MockBuffer(fmt, sizeof(PackedStruct))
@testcase
def partially_packed_struct(fmt):
"""
Assuming int is four bytes:
>>> partially_packed_struct("^c@i^ci")
>>> partially_packed_struct("@ci^ci")
>>> partially_packed_struct("^c@i=ci")
>>> partially_packed_struct("@ci=ci")
>>> partially_packed_struct("ci^ci")
>>> partially_packed_struct("ci=ci")
Incorrectly aligned accesses won't work:
>>> partially_packed_struct("^cici")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch; next field is at offset 1 but 4 expected
>>> partially_packed_struct("=cibi")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch; next field is at offset 1 but 4 expected
"""
cdef object[PartiallyPackedStruct] buf = MockBuffer(
fmt, sizeof(PartiallyPackedStruct))
@testcase
def partially_packed_struct_2(fmt):
"""
Assuming int is four bytes:
>>> partially_packed_struct_2("^ccxxxici")
>>> partially_packed_struct_2("^ccxxxi^ci")
>>> partially_packed_struct_2("c=cxxxi^ci")
>>> partially_packed_struct_2("c^cxxxi^ci")
>>> partially_packed_struct_2("c^cxxxi=ci")
>>> partially_packed_struct_2("ccxxx^i@c^i")
Incorrectly aligned accesses won't work:
>>> partially_packed_struct_2("ccxxxici")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch; next field is at offset 8 but 5 expected
>>> partially_packed_struct_2("ccici")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch; next field is at offset 4 but 5 expected
"""
cdef object[PartiallyPackedStruct2] buf = MockBuffer(
fmt, sizeof(PartiallyPackedStruct2))
# TODO: empty struct # TODO: empty struct
# TODO: Incomplete structs # TODO: Incomplete structs
# TODO: mixed structs # TODO: mixed structs
...@@ -210,6 +210,30 @@ try: ...@@ -210,6 +210,30 @@ try:
dtype=[('x', '!f8'), ('y', '!f8')]) dtype=[('x', '!f8'), ('y', '!f8')])
""" """
if np.__version__ >= '1.6':
__doc__ += u"""
The following expose bugs in Numpy (versions prior to 2011-04-02):
>>> print(test_partially_packed_align(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')], align=True))))
array([(22, 23, (24, 25), 26)],
dtype=[('a', '|i1'), ('', '|V3'), ('b', '!i4'), ('sub', [('f0', '|i1'), ('f1', '!i4')]), ('', '|V3'), ('c', '!i4')])
>>> print(test_partially_packed_align_2(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('c', 'b'), ('sub', np.dtype('b,i', align=True))]))))
array([(22, 23, 24, (27, 28))],
dtype=[('a', '|i1'), ('b', '!i4'), ('c', '|i1'), ('sub', [('f0', '|i1'), ('', '|V3'), ('f1', '!i4')])])
>>> print(test_partially_packed_align(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')], align=False)))) #doctest: +ELLIPSIS
Traceback (most recent call last):
...
ValueError: ...
>>> print(test_partially_packed_align_2(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('c', 'b'), ('sub', np.dtype('b,i', align=False))])))) #doctest: +ELLIPSIS
Traceback (most recent call last):
...
ValueError: ...
"""
except: except:
__doc__ = u"" __doc__ = u""
...@@ -402,6 +426,18 @@ cdef struct UnpackedStruct: ...@@ -402,6 +426,18 @@ cdef struct UnpackedStruct:
char a char a
int b int b
cdef struct PartiallyPackedStruct:
char a
int b
PackedStruct sub
int c
cdef packed struct PartiallyPackedStruct2:
char a
int b
char c
UnpackedStruct sub
def test_packed_align(np.ndarray[PackedStruct] arr): def test_packed_align(np.ndarray[PackedStruct] arr):
arr[0].a = 22 arr[0].a = 22
arr[0].b = 23 arr[0].b = 23
...@@ -412,6 +448,22 @@ def test_unpacked_align(np.ndarray[UnpackedStruct] arr): ...@@ -412,6 +448,22 @@ def test_unpacked_align(np.ndarray[UnpackedStruct] arr):
arr[0].b = 23 arr[0].b = 23
return repr(arr).replace('<', '!').replace('>', '!') return repr(arr).replace('<', '!').replace('>', '!')
def test_partially_packed_align(np.ndarray[PartiallyPackedStruct] arr):
arr[0].a = 22
arr[0].b = 23
arr[0].sub.a = 24
arr[0].sub.b = 25
arr[0].c = 26
return repr(arr).replace('<', '!').replace('>', '!')
def test_partially_packed_align_2(np.ndarray[PartiallyPackedStruct2] arr):
arr[0].a = 22
arr[0].b = 23
arr[0].c = 24
arr[0].sub.a = 27
arr[0].sub.b = 28
return repr(arr).replace('<', '!').replace('>', '!')
def test_complextypes(): def test_complextypes():
cdef np.complex64_t x64 = 1, y64 = 1j cdef np.complex64_t x64 = 1, y64 = 1j
cdef np.complex128_t x128 = 1, y128 = 1j cdef np.complex128_t x128 = 1, y128 = 1j
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment