Skip to content

C
cython
Commit 4efa79b6 authored by Dag Sverre Seljebotn's avatar Dag Sverre Seljebotn
Browse files
Options

Merge branch 'bug/3118-align'

parents 269fb2b8 fcd59a76
Hide whitespace changes
Inline Side-by-side
Showing with 201 additions and 24 deletions
Cython/Compiler/Buffer.py
View file @ 4efa79b6
......@@ -747,7 +747,8 @@ typedef struct {
int new_count, enc_count;
int is_complex;
char enc_type;
char packmode;
char new_packmode;
char enc_packmode;
} __Pyx_BufFmt_Context;
static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx,
......@@ -762,7 +763,8 @@ static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx,
ctx->head->field = &ctx->root;
ctx->fmt_offset = 0;
ctx->head->parent_offset = 0;
ctx->packmode = '@';
ctx->new_packmode = '@';
ctx->enc_packmode = '@';
ctx->new_count = 1;
ctx->enc_count = 0;
ctx->enc_type = 0;
......@@ -936,12 +938,12 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
__Pyx_StructField* field = ctx->head->field;
__Pyx_TypeInfo* type = field->type;
if (ctx->packmode == '@' || ctx->packmode == '^') {
if (ctx->enc_packmode == '@' || ctx->enc_packmode == '^') {
size = __Pyx_BufFmt_TypeCharToNativeSize(ctx->enc_type, ctx->is_complex);
} else {
size = __Pyx_BufFmt_TypeCharToStandardSize(ctx->enc_type, ctx->is_complex);
}
if (ctx->packmode == '@') {
if (ctx->enc_packmode == '@') {
int align_at = __Pyx_BufFmt_TypeCharToAlignment(ctx->enc_type, ctx->is_complex);
int align_mod_offset;
if (align_at == 0) return -1;
......@@ -1008,14 +1010,6 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
return 0;
}
static int __Pyx_BufFmt_FirstPack(__Pyx_BufFmt_Context* ctx) {
if (ctx->enc_type != 0 || ctx->packmode != '@') {
PyErr_SetString(PyExc_ValueError, "Buffer packing mode currently only allowed at beginning of format string (this is a defect)");
return -1;
}
return 0;
}
static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) {
int got_Z = 0;
while (1) {
......@@ -1041,8 +1035,7 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
PyErr_SetString(PyExc_ValueError, "Little-endian buffer not supported on big-endian compiler");
return NULL;
}
if (__Pyx_BufFmt_FirstPack(ctx) == -1) return NULL;
ctx->packmode = '=';
ctx->new_packmode = '=';
++ts;
break;
case '>':
......@@ -1051,15 +1044,13 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
PyErr_SetString(PyExc_ValueError, "Big-endian buffer not supported on little-endian compiler");
return NULL;
}
if (__Pyx_BufFmt_FirstPack(ctx) == -1) return NULL;
ctx->packmode = '=';
ctx->new_packmode = '=';
++ts;
break;
case '=':
case '@':
case '^':
if (__Pyx_BufFmt_FirstPack(ctx) == -1) return NULL;
ctx->packmode = *ts++;
ctx->new_packmode = *ts++;
break;
case 'T': /* substruct */
{
......@@ -1090,6 +1081,7 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
ctx->new_count = 1;
ctx->enc_count = 0;
ctx->enc_type = 0;
ctx->enc_packmode = ctx->new_packmode;
++ts;
break;
case 'Z':
......@@ -1103,13 +1095,15 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
case 'l': case 'L': case 'q': case 'Q':
case 'f': case 'd': case 'g':
case 'O':
if (ctx->enc_type == *ts && got_Z == ctx->is_complex) {
if (ctx->enc_type == *ts && got_Z == ctx->is_complex &&
ctx->enc_packmode == ctx->new_packmode) {
/* Continue pooling same type */
ctx->enc_count += ctx->new_count;
} else {
/* New type */
if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL;
ctx->enc_count = ctx->new_count;
ctx->enc_packmode = ctx->new_packmode;
ctx->enc_type = *ts;
ctx->is_complex = got_Z;
}
......@@ -1117,7 +1111,7 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
ctx->new_count = 1;
got_Z = 0;
break;
case ':':
case ':':
++ts;
while(*ts != ':') ++ts;
++ts;
......
tests/run/bufaccess.pyx
View file @ 4efa79b6
......@@ -1281,6 +1281,16 @@ cdef struct NestedStruct:
SmallStruct y
int z
cdef packed struct PackedStruct:
char a
int b
cdef struct NestedPackedStruct:
char a
int b
PackedStruct sub
int c
cdef class MyStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef MyStruct* s
......@@ -1301,6 +1311,26 @@ cdef class NestedStructMockBuffer(MockBuffer):
cdef get_itemsize(self): return sizeof(NestedStruct)
cdef get_default_format(self): return b"2T{ii}i"
cdef class PackedStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef PackedStruct* s
s = <PackedStruct*>buf;
s.a, s.b = value
return 0
cdef get_itemsize(self): return sizeof(PackedStruct)
cdef get_default_format(self): return b"^ci"
cdef class NestedPackedStructMockBuffer(MockBuffer):
cdef int write(self, char* buf, object value) except -1:
cdef NestedPackedStruct* s
s = <NestedPackedStruct*>buf;
s.a, s.b, s.sub.a, s.sub.b, s.c = value
return 0
cdef get_itemsize(self): return sizeof(NestedPackedStruct)
cdef get_default_format(self): return b"ci^ci@i"
@testcase
def basic_struct(object[MyStruct] buf):
"""
......@@ -1325,6 +1355,35 @@ def nested_struct(object[NestedStruct] buf):
"""
print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z
@testcase
def packed_struct(object[PackedStruct] buf):
"""
See also buffmt.pyx
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)]))
1 2
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c^i}"))
1 2
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c=i}"))
1 2
"""
print buf[0].a, buf[0].b
@testcase
def nested_packed_struct(object[NestedPackedStruct] buf):
"""
See also buffmt.pyx
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
1 2 3 4 5
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ci^ci@i"))
1 2 3 4 5
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i"))
1 2 3 4 5
"""
print buf[0].a, buf[0].b, buf[0].sub.a, buf[0].sub.b, buf[0].c
cdef struct LongComplex:
long double real
long double imag
......
tests/run/buffmt.pyx
View file @ 4efa79b6
......@@ -155,7 +155,7 @@ def char3int(fmt):
>>> char3int("c3i")
>>> char3int("ci2i")
#TODO > char3int("c@i@2i")
>>> char3int("c@i@2i")
Extra pad bytes (assuming int size is 4 or more)
>>> char3int("cxiii")
......@@ -169,7 +169,7 @@ def char3int(fmt):
...
ValueError: Buffer dtype mismatch; next field is at offset 1 but 4 expected
#TODO char3int("=cxxx@iii")
>>> char3int("=cxxx@iii")
Error:
>>> char3int("cii")
......@@ -277,11 +277,25 @@ cdef packed struct PackedSubStruct:
char x
int y
cdef struct UnpackedSubStruct:
char x
int y
cdef packed struct PackedStruct:
char a
int b
PackedSubStruct sub
cdef struct PartiallyPackedStruct:
char a
int b
PackedSubStruct sub
cdef packed struct PartiallyPackedStruct2:
char a
UnpackedSubStruct sub
char b
int c
@testcase
def packed_struct(fmt):
......@@ -291,12 +305,13 @@ def packed_struct(fmt):
>>> packed_struct("^cici")
>>> packed_struct("=cibi")
However aligned access won't work:
>>> packed_struct("^c@i^ci")
Traceback (most recent call last):
...
ValueError: Buffer packing mode currently only allowed at beginning of format string (this is a defect)
ValueError: Buffer dtype mismatch; next field is at offset 4 but 1 expected
However aligned access won't work:
>>> packed_struct("@cici")
Traceback (most recent call last):
...
......@@ -305,6 +320,63 @@ def packed_struct(fmt):
"""
cdef object[PackedStruct] buf = MockBuffer(fmt, sizeof(PackedStruct))
@testcase
def partially_packed_struct(fmt):
"""
Assuming int is four bytes:
>>> partially_packed_struct("^c@i^ci")
>>> partially_packed_struct("@ci^ci")
>>> partially_packed_struct("^c@i=ci")
>>> partially_packed_struct("@ci=ci")
>>> partially_packed_struct("ci^ci")
>>> partially_packed_struct("ci=ci")
Incorrectly aligned accesses won't work:
>>> partially_packed_struct("^cici")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch; next field is at offset 1 but 4 expected
>>> partially_packed_struct("=cibi")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch; next field is at offset 1 but 4 expected
"""
cdef object[PartiallyPackedStruct] buf = MockBuffer(
fmt, sizeof(PartiallyPackedStruct))
@testcase
def partially_packed_struct_2(fmt):
"""
Assuming int is four bytes:
>>> partially_packed_struct_2("^ccxxxici")
>>> partially_packed_struct_2("^ccxxxi^ci")
>>> partially_packed_struct_2("c=cxxxi^ci")
>>> partially_packed_struct_2("c^cxxxi^ci")
>>> partially_packed_struct_2("c^cxxxi=ci")
>>> partially_packed_struct_2("ccxxx^i@c^i")
Incorrectly aligned accesses won't work:
>>> partially_packed_struct_2("ccxxxici")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch; next field is at offset 8 but 5 expected
>>> partially_packed_struct_2("ccici")
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch; next field is at offset 4 but 5 expected
"""
cdef object[PartiallyPackedStruct2] buf = MockBuffer(
fmt, sizeof(PartiallyPackedStruct2))
# TODO: empty struct
# TODO: Incomplete structs
# TODO: mixed structs
tests/run/numpy_test.pyx
View file @ 4efa79b6
......@@ -210,6 +210,30 @@ try:
dtype=[('x', '!f8'), ('y', '!f8')])
"""
if np.__version__ >= '1.6':
__doc__ += u"""
The following expose bugs in Numpy (versions prior to 2011-04-02):
>>> print(test_partially_packed_align(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')], align=True))))
array([(22, 23, (24, 25), 26)],
dtype=[('a', '|i1'), ('', '|V3'), ('b', '!i4'), ('sub', [('f0', '|i1'), ('f1', '!i4')]), ('', '|V3'), ('c', '!i4')])
>>> print(test_partially_packed_align_2(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('c', 'b'), ('sub', np.dtype('b,i', align=True))]))))
array([(22, 23, 24, (27, 28))],
dtype=[('a', '|i1'), ('b', '!i4'), ('c', '|i1'), ('sub', [('f0', '|i1'), ('', '|V3'), ('f1', '!i4')])])
>>> print(test_partially_packed_align(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')], align=False)))) #doctest: +ELLIPSIS
Traceback (most recent call last):
...
ValueError: ...
>>> print(test_partially_packed_align_2(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('c', 'b'), ('sub', np.dtype('b,i', align=False))])))) #doctest: +ELLIPSIS
Traceback (most recent call last):
...
ValueError: ...
"""
except:
__doc__ = u""
......@@ -402,6 +426,18 @@ cdef struct UnpackedStruct:
char a
int b
cdef struct PartiallyPackedStruct:
char a
int b
PackedStruct sub
int c
cdef packed struct PartiallyPackedStruct2:
char a
int b
char c
UnpackedStruct sub
def test_packed_align(np.ndarray[PackedStruct] arr):
arr[0].a = 22
arr[0].b = 23
......@@ -412,6 +448,22 @@ def test_unpacked_align(np.ndarray[UnpackedStruct] arr):
arr[0].b = 23
return repr(arr).replace('<', '!').replace('>', '!')
def test_partially_packed_align(np.ndarray[PartiallyPackedStruct] arr):
arr[0].a = 22
arr[0].b = 23
arr[0].sub.a = 24
arr[0].sub.b = 25
arr[0].c = 26
return repr(arr).replace('<', '!').replace('>', '!')
def test_partially_packed_align_2(np.ndarray[PartiallyPackedStruct2] arr):
arr[0].a = 22
arr[0].b = 23
arr[0].c = 24
arr[0].sub.a = 27
arr[0].sub.b = 28
return repr(arr).replace('<', '!').replace('>', '!')
def test_complextypes():
cdef np.complex64_t x64 = 1, y64 = 1j
cdef np.complex128_t x128 = 1, y128 = 1j
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7