Commit 95b3e447 authored by Robert Bradshaw's avatar Robert Bradshaw

Merge remote-tracking branch 'andreasvc/master' into arrays

parents ee0f7bf6 16893be6
"""
array.pxd
Cython interface to Python's array.array module.
* 1D contiguous data view
* tools for fast array creation, maximum C-speed and handiness
* suitable as allround light weight auto-array within Cython code too
Usage:
>>> cimport array
Usage through Cython buffer interface (Py2.3+):
>>> def f(arg1, unsigned i, double dx)
... array.array[double] a = arg1
... a[i] += dx
Fast C-level new_array(_zeros), resize_array, copy_array, .length,
zero_array
cdef array.array[double] k = array.copy(d)
cdef array.array[double] n = array.array(d, d.length * 2 )
cdef array.array[double] m = array.zeros_like(FLOAT_TEMPLATE)
array.resize(f, 200000)
Zero overhead with naked data pointer views by union:
_f, _d, _i, _c, _u, ...
=> Original C array speed + Python dynamic memory management
cdef array.array a = inarray
if
a._d[2] += 0.66 # use as double array without extra casting
float *subview = vector._f + 10 # starting from 10th element
unsigned char *subview_buffer = vector._B + 4
Suitable as lightweight arrays intra Cython without speed penalty.
Replacement for C stack/malloc arrays; no trouble with refcounting,
mem.leaks; seamless Python compatibility, buffer() optional
IMPORTANT: arrayarray.h (arrayobject, arraydescr) is not part of
the official Python C-API so far; arrayarray.h is located
next to this file copy it to PythonXX/include or local or
somewhere on your -I path
last changes: 2009-05-15 rk
: 2009-12-06 bp
: 2012-05-02 andreasvc
"""
from libc cimport stdlib
from libc.string cimport strcat, strncat, \
memset, memchr, memcmp, memcpy, memmove
from cpython.ref cimport PyTypeObject
from cpython.exc cimport PyErr_BadArgument
cdef extern from "arrayarray.h":
ctypedef class array.array [object arrayobject]
ctypedef object GETF(array a, Py_ssize_t ix)
ctypedef object SETF(array a, Py_ssize_t ix, object o)
ctypedef struct arraydescr: # [object arraydescr]:
int typecode
int itemsize
GETF getitem # PyObject * (*getitem)(struct arrayobject *, Py_ssize_t);
SETF setitem # int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *);
ctypedef class array.array [object arrayobject]:
cdef __cythonbufferdefaults__ = {'ndim' : 1, 'mode':'c'}
cdef:
PyTypeObject* ob_type
Py_ssize_t length # == ob_size (by union)
arraydescr* ob_descr # struct arraydescr *ob_descr;
# views of ob_item:
float* _f # direct float pointer access to buffer
double* _d # double ...
int* _i
unsigned *_I
unsigned char *_B
signed char *_b
char *_c
unsigned long *_L
long *_l
short *_h
unsigned short *_H
Py_UNICODE *_u
void* _v
def __getbuffer__(array self, Py_buffer* info, int flags):
# This implementation of getbuffer is geared towards Cython
# requirements, and does not yet fullfill the PEP.
# In particular strided access is always provided regardless
# of flags
cdef unsigned rows, columns, itemsize
info.suboffsets = NULL
info.buf = self._c
info.readonly = 0
info.ndim = 1
info.itemsize = itemsize = self.ob_descr.itemsize # e.g. sizeof(float)
info.strides = <Py_ssize_t*> \
stdlib.malloc(sizeof(Py_ssize_t) * info.ndim * 2 + 2)
info.shape = info.strides + 1
info.shape[0] = self.length # number of items
info.strides[0] = info.itemsize
info.format = <char*>(info.strides + 2 * info.ndim)
info.format[0] = self.ob_descr.typecode
info.format[1] = 0
info.obj = self
def __releasebuffer__(array self, Py_buffer* info):
#if PyArray_HASFIELDS(self):
# stdlib.free(info.format)
#if sizeof(npy_intp) != sizeof(Py_ssize_t):
stdlib.free(info.strides)
array newarrayobject(PyTypeObject* type, Py_ssize_t size,
arraydescr *descr)
# fast resize/realloc
# not suitable for small increments; reallocation 'to the point'
int resize(array self, Py_ssize_t n)
# efficient for small increments (not in Py2.3-)
int resize_smart(array self, Py_ssize_t n)
cdef inline array clone(array template, Py_ssize_t length, bint zero):
""" fast creation of a new array, given a template array.
type will be same as template.
if zero is true, new array will be initialized with zeroes."""
cdef array op
op = newarrayobject(template.ob_type, length, template.ob_descr)
if zero and op is not None:
memset(op._c, 0, length * op.ob_descr.itemsize)
return op
cdef inline array copy(array self):
""" make a copy of an array. """
cdef array op
op = newarrayobject(self.ob_type, self.length, self.ob_descr)
memcpy(op._c, self._c, op.length * op.ob_descr.itemsize)
return op
cdef inline int extend_buffer(array self, char* stuff, Py_ssize_t n):
""" efficent appending of new stuff of same type
(e.g. of same array type)
n: number of elements (not number of bytes!) """
cdef Py_ssize_t itemsize = self.ob_descr.itemsize
cdef Py_ssize_t orgsize = self.length
if resize_smart(self, orgsize + n) == -1:
return -1
memcpy(self._c + orgsize * itemsize, stuff, n * itemsize)
cdef inline int extend(array self, array other):
""" extend array with data from another array; types must match. """
if self.ob_descr.typecode != self.ob_descr.typecode:
PyErr_BadArgument()
return -1
return extend_buffer(self, other._c, other.length)
cdef inline void zero(array op):
""" set all elements of array to zero. """
memset(op._c, 0, op.length * op.ob_descr.itemsize)
/* arrayarray.h
artificial C-API for Python's
<array.array> type.
copy this file to your -I path, e.g. .../pythonXX/include
See array.pxd next to this file
last changes: 2009-05-15 rk
2012-05-02 andreasvc
*/
#ifndef _ARRAYARRAY_H
#define _ARRAYARRAY_H
#include <Python.h>
struct arrayobject; /* Forward */
/* All possible arraydescr values are defined in the vector "descriptors"
* below. That's defined later because the appropriate get and set
* functions aren't visible yet.
*/
typedef struct arraydescr {
int typecode;
int itemsize;
PyObject * (*getitem)(struct arrayobject *, Py_ssize_t);
int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *);
#if PY_VERSION_HEX >= 0x03000000
char *formats;
#endif
} arraydescr;
typedef struct arrayobject {
PyObject_HEAD
union {
Py_ssize_t ob_size, length;
};
union {
char *ob_item;
float *_f;
double *_d;
int *_i;
unsigned *_I;
unsigned char *_B;
signed char *_b;
char *_c;
unsigned long *_L;
long *_l;
short *_h;
unsigned short *_H;
Py_UNICODE *_u;
void *_v;
};
#if PY_VERSION_HEX >= 0x02040000
Py_ssize_t allocated;
#endif
struct arraydescr *ob_descr;
#if PY_VERSION_HEX >= 0x02040000
PyObject *weakreflist; /* List of weak references */
#if PY_VERSION_HEX >= 0x03000000
int ob_exports; /* Number of exported buffers */
#endif
#endif
} arrayobject;
#ifndef NO_NEWARRAY_INLINE
/*
*
* fast creation of a new array
*/
inline PyObject * newarrayobject(PyTypeObject *type, Py_ssize_t size,
struct arraydescr *descr) {
arrayobject *op;
size_t nbytes;
if (size < 0) {
PyErr_BadInternalCall();
return NULL;
}
nbytes = size * descr->itemsize;
/* Check for overflow */
if (nbytes / descr->itemsize != (size_t)size) {
return PyErr_NoMemory();
}
op = (arrayobject *) type->tp_alloc(type, 0);
if (op == NULL) {
return NULL;
}
op->ob_descr = descr;
#if !( PY_VERSION_HEX < 0x02040000 )
op->allocated = size;
op->weakreflist = NULL;
#endif
Py_SIZE(op) = size;
if (size <= 0) {
op->ob_item = NULL;
}
else {
op->ob_item = PyMem_NEW(char, nbytes);
if (op->ob_item == NULL) {
Py_DECREF(op);
return PyErr_NoMemory();
}
}
return (PyObject *) op;
}
#else
PyObject* newarrayobject(PyTypeObject *type, Py_ssize_t size,
struct arraydescr *descr);
#endif /* ifndef NO_NEWARRAY_INLINE */
/* fast resize (reallocation to the point)
not designed for filing small increments (but for fast opaque array apps) */
int resize(arrayobject *self, Py_ssize_t n) {
void *item=self->ob_item;
PyMem_Resize(item, char, (size_t)(n * self->ob_descr->itemsize));
if (item == NULL) {
PyErr_NoMemory();
return -1;
}
self->ob_item = item;
self->ob_size = n;
#if PY_VERSION_HEX >= 0x02040000
self->allocated = n;
#endif
return 0;
}
/* suitable for small increments; over allocation 50% ;
Remains non-smart in Python 2.3- ; but exists for compatibility */
int resize_smart(arrayobject *self, Py_ssize_t n) {
#if PY_VERSION_HEX >= 0x02040000
void *item=self->ob_item;
Py_ssize_t newsize;
if (n < self->allocated) {
if (n*4 > self->allocated) {
self->ob_size = n;
return 0;
}
}
newsize = n * 3 / 2 + 1;
PyMem_Resize(item, char, (size_t)(newsize * self->ob_descr->itemsize));
if (item == NULL) {
PyErr_NoMemory();
return -1;
}
self->ob_item = item;
self->ob_size = n;
self->allocated = newsize;
return 0;
#else
return resize(self, n) /* Python 2.3 has no 'allocated' */
#endif
}
#endif
/* _ARRAYARRAY_H */
...@@ -76,12 +76,17 @@ EXT_DEP_MODULES = { ...@@ -76,12 +76,17 @@ EXT_DEP_MODULES = {
'tag:numpy' : 'numpy', 'tag:numpy' : 'numpy',
'tag:pstats': 'pstats', 'tag:pstats': 'pstats',
'tag:posix' : 'posix', 'tag:posix' : 'posix',
'tag:array' : 'array',
} }
def update_numpy_extension(ext): def update_numpy_extension(ext):
import numpy import numpy
ext.include_dirs.append(numpy.get_include()) ext.include_dirs.append(numpy.get_include())
def update_pyarray_extension(ext):
ext.include_dirs.append(
os.path.join(os.path.dirname(DISTDIR), "Cython/Includes/cpython"))
def update_openmp_extension(ext): def update_openmp_extension(ext):
ext.openmp = True ext.openmp = True
language = ext.language language = ext.language
...@@ -157,6 +162,7 @@ EXCLUDE_EXT = object() ...@@ -157,6 +162,7 @@ EXCLUDE_EXT = object()
EXT_EXTRAS = { EXT_EXTRAS = {
'tag:numpy' : update_numpy_extension, 'tag:numpy' : update_numpy_extension,
'tag:array' : update_pyarray_extension,
'tag:openmp': update_openmp_extension, 'tag:openmp': update_openmp_extension,
} }
......
# tag: array
import array # Python builtin module
from cpython cimport array # array.pxd / arrayarray.h
a = array.array('f', [1.0, 2.0, 3.0])
def test_len(a):
"""
>>> a = array.array('f', [1.0, 2.0, 3.0])
>>> len(a)
3
>>> test_len(a)
3L
>>> assert len(a) == test_len(a)
"""
cdef array.array ca = a # for C-fast array usage
return ca.length
def test_copy(a):
"""
>>> a = array.array('f', [1.0, 2.0, 3.0])
>>> test_copy(a)
array('f', [1.0, 2.0, 3.0])
>>> assert a == test_copy(a)
"""
cdef array.array ca = a
cdef array.array b
b = array.copy(a)
a[2] = 3.5
assert b[2] != a[2]
return b
def test_fast_access(a):
"""
>>> a = array.array('f', [1.0, 2.0, 3.0])
>>> a[2]
3.5
>>> test_fast_access(a)
"""
cdef array.array ca = a
assert ca._f[1] == 2.0, ca._f[1]
assert ca._c[:5] == b'\x00\x00\x80?\x00', ca._c[:5]
ca._f[1] += 2.0
assert ca._f[1] == 4.0
def test_new_zero(a):
"""
>>> a = array.array('f', [1.0, 2.0, 3.0])
>>> test_new_zero(a)
array('f', [0.0, 0.0, 0.0])
"""
cdef array.array cb = array.clone(a, len(a), True)
assert cb.length == len(a)
return cb
def test_set_zero(a):
"""
>>> a = array.array('f', [1.0, 2.0, 3.0])
>>> test_set_zero(a)
array('f', [0.0, 0.0, 0.0])
"""
cdef array.array cb = array.copy(a)
array.zero(cb)
assert a[1] != 0.0, a
assert cb[1] == 0.0, cb
return cb
def test_resize(a):
"""
>>> a = array.array('f', [1.0, 2.0, 3.0])
>>> test_resize(a)
"""
cdef array.array cb = array.copy(a)
array.resize(cb, 10)
for i in range(10):
cb._f[i] = i
assert cb.length == 10
assert cb[9] == cb[-1] == cb._f[9] == 9
def test_view():
"""
>>> a = array.array('f', [1.0, 2.0, 3.0])
>>> test_view()
"""
a = array.array('i', [1, 2, 3])
cdef object[int] ca = a
assert ca[0] == 1
assert ca[2] == 3
def test_extend():
"""
>>> a = array.array('f', [1.0, 2.0, 3.0])
>>> test_extend()
"""
cdef array.array ca = array.array('i', [1, 2, 3])
cdef array.array cb = array.array('i', range(4, 6))
array.extend(ca, cb)
assert list(ca) == range(1, 6), list(ca)
def test_likes(a):
"""
>>> a = array.array('f', [1.0, 2.0, 3.0])
>>> test_likes(a)
array('f', [0.0, 0.0, 0.0])
"""
cdef array.array z = array.clone(a, len(a), True)
cdef array.array e = array.clone(a, len(a), False)
assert e.length == len(a)
return z
def test_extend_buffer():
"""
>>> a = array.array('f', [1.0, 2.0, 3.0])
>>> test_extend_buffer()
array('c', 'abcdefghij')
"""
cdef array.array ca = array.array('c', "abcdef")
cdef char* s = "ghij"
array.extend_buffer(ca, s, len(s)) # or use stdlib.strlen
assert ca._c[9] == 'j'
assert len(ca) == 10
return ca
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment