Commit 556722cb authored by Mark Florisson's avatar Mark Florisson

Utility Code loader + memview python3 compat

parent c8be949c
......@@ -1090,9 +1090,6 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
}
static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) {
if (!ts) {
PyErr_SetString(PyExc_ValueError, "Got NULL buffer format");
}
int got_Z = 0;
while (1) {
switch(*ts) {
......
cimport cython
cdef class UtilityCode:
cdef class UtilityCodeBase(object):
pass
cdef class UtilityCode(UtilityCodeBase):
cdef public object proto
cdef public object impl
cdef public object init
......
......@@ -8,7 +8,10 @@ cython.declare(re=object, Naming=object, Options=object, StringEncoding=object,
Utils=object, SourceDescriptor=object, StringIOTree=object,
DebugFlags=object, none_or_sub=object, basestring=object)
import os
import re
import codecs
import Naming
import Options
import StringEncoding
......@@ -16,6 +19,7 @@ from Cython import Utils
from Scanning import SourceDescriptor
from Cython.StringIOTree import StringIOTree
import DebugFlags
import Errors
from Cython.Utils import none_or_sub
try:
......@@ -38,17 +42,130 @@ uncachable_builtins = [
'WindowsError',
]
class UtilityCode(object):
class UtilityCodeBase(object):
is_cython_utility = False
_utility_cache = {}
@classmethod
def _add_utility(self, utility, type, lines, begin_lineno):
if utility:
code = '\n' * begin_lineno + ''.join(lines)
if type == 'Proto':
utility[0] = code
else:
utility[1] = code
@classmethod
def load_utilities_from_file(cls, path):
utilities = cls._utility_cache.get(path)
if utilities:
return utilities
Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
filename = os.path.join(Cython_dir, "Utility", path)
f = codecs.open(filename, encoding='UTF-8')
_, ext = os.path.splitext(path)
if ext in ('.pyx', '.py', '.pxd', '.pxi'):
comment = '#'
else:
comment = '//'
regex = r'%s\s*Utility(Proto|Code)\s*:\s*((\w|\.)+)\s*' % comment
flags = re.DOTALL
utilities = {}
lines = []
utility = type = None
begin_lineno = 0
for lineno, line in enumerate(f):
m = re.search(regex, line)
if m:
cls._add_utility(utility, type, lines, begin_lineno)
begin_lineno = lineno + 1
type, name = m.group(1), m.group(2)
utility = utilities.setdefault(name, [None, None])
utilities[name] = utility
lines = []
else:
lines.append(line)
if not utility:
raise ValueError("Empty utility code file")
# Don't forget to add the last utility code
cls._add_utility(utility, type, lines, begin_lineno)
f.close()
cls._utility_cache[path] = utilities
return utilities
@classmethod
def load_utility_from_file(cls, path, util_code_name, proto_fmt_dict=None,
impl_fmt_dict=None, *args, **kwargs):
"""
Load a utility code from a file specified by path (relative to
Cython/Utility) and name util_code_name.
Utilities in the file can be specified as follows:
# UtilityProto: name
# UtilityImpl: name
for prototypes and implementation respectively. For non-python or
-cython files a // comment should be used instead.
proto_fmt_dict and impl_format_dict can optionally be given to perform
any substitutions.
If the @cname decorator is not used and this is a CythonUtilityCode,
one should pass in the 'name' keyword argument to be used for name
mangling of such entries.
"""
proto, impl = cls.load_utility_as_string(path, util_code_name)
if proto:
if proto_fmt_dict:
proto = proto % proto_fmt_dict
kwargs['proto'] = proto
if impl:
if impl_fmt_dict:
impl = impl % impl_fmt_dict
kwargs['impl'] = impl
if 'name' not in kwargs:
kwargs['name'] = os.path.splitext(path)[0]
return cls(*args, **kwargs)
@classmethod
def load_utility_as_string(cls, path, util_code_name):
"""
Load a utility code as a string. Returns (proto, implementation)
"""
utilities = cls.load_utilities_from_file(path)
return utilities[util_code_name]
def __str__(self):
return "<%s(%s)" % (type(self).__name__, self.name)
class UtilityCode(UtilityCodeBase):
# Stores utility code to add during code generation.
#
# See GlobalState.put_utility_code.
#
# hashes/equals by instance
is_cython_utility = False
def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
proto_block='utility_code_proto'):
proto_block='utility_code_proto', name=None):
# proto_block: Which code block to dump prototype in. See GlobalState.
self.proto = proto
self.impl = impl
......@@ -58,6 +175,7 @@ class UtilityCode(object):
self._cache = {}
self.specialize_list = []
self.proto_block = proto_block
self.name = name
def get_tree(self):
pass
......
......@@ -5,6 +5,7 @@ from Errors import error
from Scanning import StringSourceDescriptor
import Options
import Buffer
import MemoryView
class CythonScope(ModuleScope):
is_cython_builtin = 1
......@@ -100,11 +101,12 @@ def create_cython_scope(context, create_testscope):
return scope
cython_testscope_utility_code = CythonUtilityCode(u"""
@cname('__pyx_testscope')
cdef object _testscope(int value):
return "hello from cython scope, value=%d" % value
""")
# Load test utilities for the cython scope
def load_testscope_utility(cython_util_name, *args, **kwargs):
return CythonUtilityCode.load_utility_from_file(
"TestCythonScope.pyx", cython_util_name, *args, **kwargs)
undecorated_methods_protos = UtilityCode(proto=u"""
/* These methods are undecorated and have therefore no prototype */
......@@ -116,237 +118,18 @@ undecorated_methods_protos = UtilityCode(proto=u"""
PyObject *self, PyObject *value);
""")
test_cython_utility_dep = CythonUtilityCode(u"""
@cname('__pyx_test_dep')
cdef test_dep(obj):
print 'test_dep', obj
""")
cython_test_extclass_utility_code = CythonUtilityCode(
name="TestClassUtilityCode",
prefix="__pyx_prefix_TestClass_",
requires=[undecorated_methods_protos, test_cython_utility_dep],
impl=u"""
cdef extern from *:
cdef object __pyx_test_dep(object)
@cname('__pyx_TestClass')
cdef class TestClass(object):
cdef public int value
def __init__(self, int value):
self.value = value
def __str__(self):
return 'TestClass(%d)' % self.value
cdef cdef_method(self, int value):
print 'Hello from cdef_method', value
cpdef cpdef_method(self, int value):
print 'Hello from cpdef_method', value
def def_method(self, int value):
print 'Hello from def_method', value
@cname('cdef_cname')
cdef cdef_cname_method(self, int value):
print "Hello from cdef_cname_method", value
@cname('cpdef_cname')
cpdef cpdef_cname_method(self, int value):
print "Hello from cpdef_cname_method", value
@cname('def_cname')
def def_cname_method(self, int value):
print "Hello from def_cname_method", value
@cname('__pyx_test_call_other_cy_util')
cdef test_call(obj):
print 'test_call'
__pyx_test_dep(obj)
@cname('__pyx_TestClass_New')
cdef _testclass_new(int value):
return TestClass(value)
""")
cythonview_testscope_utility_code = CythonUtilityCode(u"""
@cname('__pyx_view_testscope')
cdef object _testscope(int value):
return "hello from cython.view scope, value=%d" % value
""")
memview_name = u'memoryview'
memview_typeptr_cname = '__pyx_memoryview_type'
memview_objstruct_cname = '__pyx_memoryview_obj'
view_utility_code = CythonUtilityCode(
u"""
cdef class Enum(object):
cdef object name
def __init__(self, name):
self.name = name
def __repr__(self):
return self.name
cdef strided = Enum("<strided axis packing mode>")
cdef contig = Enum("<contig axis packing mode>")
cdef follow = Enum("<follow axis packing mode>")
cdef direct = Enum("<direct axis access mode>")
cdef ptr = Enum("<ptr axis access mode>")
cdef full = Enum("<full axis access mode>")
cdef extern from *:
int __Pyx_GetBuffer(object, Py_buffer *, int)
void __Pyx_ReleaseBuffer(Py_buffer *)
@cname('__pyx_memoryview')
cdef class memoryview(object):
cdef object obj
cdef Py_buffer view
cdef int gotbuf_flag
def __cinit__(memoryview self, object obj, int flags):
self.obj = obj
__Pyx_GetBuffer(self.obj, &self.view, flags)
def __dealloc__(memoryview self):
self.obj = None
__Pyx_ReleaseBuffer(&self.view)
@cname('__pyx_memoryview_new')
cdef memoryview memoryview_cwrapper(object o, int flags):
return memoryview(o, flags)
""", name="view_code", requires=(Buffer.GetAndReleaseBufferUtilityCode(),))
cython_array_utility_code = CythonUtilityCode(u'''
cdef extern from "stdlib.h":
void *malloc(size_t)
void free(void *)
cdef extern from "Python.h":
cdef enum:
PyBUF_C_CONTIGUOUS,
PyBUF_F_CONTIGUOUS,
PyBUF_ANY_CONTIGUOUS
@cname("__pyx_array")
cdef class array:
cdef:
char *data
Py_ssize_t len
char *format
int ndim
Py_ssize_t *shape
Py_ssize_t *strides
Py_ssize_t itemsize
str mode
void (*callback_free_data)(char *data)
def __cinit__(array self, tuple shape, Py_ssize_t itemsize, char *format,
mode="c", bint allocate_buffer=True):
self.ndim = len(shape)
self.itemsize = itemsize
if not self.ndim:
raise ValueError("Empty shape tuple for cython.array")
if self.itemsize <= 0:
raise ValueError("itemsize <= 0 for cython.array")
self.format = format
self.shape = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
self.strides = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
if not self.shape or not self.strides:
raise MemoryError("unable to allocate shape or strides.")
cdef int idx
# cdef Py_ssize_t dim, stride
idx = 0
for dim in shape:
if dim <= 0:
raise ValueError("Invalid shape.")
self.shape[idx] = dim
idx += 1
stride = itemsize
if mode == "fortran":
idx = 0
for dim in shape:
self.strides[idx] = stride
stride = stride * dim
idx += 1
elif mode == "c":
idx = self.ndim-1
for dim in shape[::-1]:
self.strides[idx] = stride
stride = stride * dim
idx -= 1
else:
raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode)
self.len = stride
self.mode = mode
if allocate_buffer:
self.data = <char *>malloc(self.len)
if not self.data:
raise MemoryError("unable to allocate array data.")
else:
self.data = NULL
self.callback_free_data = NULL
cython_testscope_utility_code = load_testscope_utility("TestScope")
def __getbuffer__(self, Py_buffer *info, int flags):
test_cython_utility_dep = load_testscope_utility("TestDep")
cdef int bufmode = -1
if self.mode == b"c":
bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
elif self.mode == b"fortran":
bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
if not (flags & bufmode):
raise ValueError("Can only create a buffer that is contiguous in memory.")
info.buf = self.data
info.len = self.len
info.ndim = self.ndim
info.shape = self.shape
info.strides = self.strides
info.suboffsets = NULL
info.itemsize = self.itemsize
info.format = self.format
# we do not need to call releasebuffer
info.obj = None
cython_test_extclass_utility_code = \
load_testscope_utility("TestClass", name="TestClass",
requires=[undecorated_methods_protos,
test_cython_utility_dep])
def __releasebuffer__(array self, Py_buffer* info):
# array.__releasebuffer__ should not be called,
# because the Py_buffer's 'obj' field is set to None.
raise NotImplementedError()
cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
def __dealloc__(array self):
if self.data:
if self.callback_free_data != NULL:
self.callback_free_data(self.data)
else:
free(self.data)
self.data = NULL
if self.strides:
free(self.strides)
self.strides = NULL
if self.shape:
free(self.shape)
self.shape = NULL
self.format = NULL
self.itemsize = 0
view_utility_code = MemoryView.load_memview_cy_utility(
"MemoryView", requires=(Buffer.GetAndReleaseBufferUtilityCode(),))
@cname("__pyx_array_new")
cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode):
return array(shape, itemsize, format, mode)
''')
cython_array_utility_code = MemoryView.load_memview_cy_utility("CythonArray")
This diff is collapsed.
......@@ -498,6 +498,7 @@ class MemoryViewSliceType(PyrexType):
def check_for_null_code(self, cname):
return cname + '.memview'
class BufferType(BaseType):
#
# Delegates most attribute
......
......@@ -3,6 +3,7 @@ from Scanning import StringSourceDescriptor
import Symtab
import Naming
from Cython.Compiler import Visitor
import Code
class NonManglingModuleScope(Symtab.ModuleScope):
......@@ -43,7 +44,7 @@ class CythonUtilityCodeContext(StringParseContext):
return self.scope
class CythonUtilityCode(object):
class CythonUtilityCode(Code.UtilityCodeBase):
"""
Utility code written in the Cython language itself.
......
// UtilityProto: MemviewSliceStruct
/* memoryview slice struct */
typedef struct {
struct %(memview_struct_name)s *memview;
char *data;
Py_ssize_t shape[%(max_dims)d];
Py_ssize_t strides[%(max_dims)d];
Py_ssize_t suboffsets[%(max_dims)d];
} %(memviewslice_name)s;
// UtilityProto: MemviewSliceInit
#define __Pyx_BUF_MAX_NDIMS %(BUF_MAX_NDIMS)d
#define __Pyx_MEMVIEW_DIRECT 1
#define __Pyx_MEMVIEW_PTR 2
#define __Pyx_MEMVIEW_FULL 4
#define __Pyx_MEMVIEW_CONTIG 8
#define __Pyx_MEMVIEW_STRIDED 16
#define __Pyx_MEMVIEW_FOLLOW 32
#define __Pyx_IS_C_CONTIG 1
#define __Pyx_IS_F_CONTIG 2
static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview,
int *axes_specs, int c_or_f_flag, int ndim, __Pyx_TypeInfo *dtype,
__Pyx_BufFmt_StackElem stack[], __Pyx_memviewslice *memviewslice);
static int __Pyx_init_memviewslice(
struct __pyx_memoryview_obj *memview,
int ndim,
__Pyx_memviewslice *memviewslice);
// UtilityCode: MemviewSliceInit
static int __Pyx_ValidateAndInit_memviewslice(
struct __pyx_memoryview_obj *memview,
int *axes_specs,
int c_or_f_flag,
int ndim,
__Pyx_TypeInfo *dtype,
__Pyx_BufFmt_StackElem stack[],
__Pyx_memviewslice *memviewslice) {
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("ValidateAndInit_memviewslice");
Py_buffer *buf = &memview->view;
int stride, i, spec = 0, retval = -1;
if (!buf) goto fail;
if(memviewslice->data || memviewslice->memview) {
PyErr_SetString(PyExc_ValueError,
"memoryviewslice struct must be initialized to NULL.");
goto fail;
}
if (buf->ndim != ndim) {
PyErr_Format(PyExc_ValueError,
"Buffer has wrong number of dimensions (expected %d, got %d)",
ndim, buf->ndim);
goto fail;
}
__Pyx_BufFmt_Context ctx;
__Pyx_BufFmt_Init(&ctx, stack, dtype);
if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail;
if ((unsigned)buf->itemsize != dtype->size) {
PyErr_Format(PyExc_ValueError,
"Item size of buffer (%" PY_FORMAT_SIZE_T "d byte%s) "
"does not match size of '%s' (%" PY_FORMAT_SIZE_T "d byte%s)",
buf->itemsize,
(buf->itemsize > 1) ? "s" : "",
dtype->name,
dtype->size,
(dtype->size > 1) ? "s" : "");
goto fail;
}
if (!buf->strides) {
PyErr_SetString(PyExc_ValueError,
"buffer does not supply strides necessary for memoryview.");
goto fail;
}
for(i=0; i<ndim; i++) {
spec = axes_specs[i];
if (spec & __Pyx_MEMVIEW_CONTIG) {
if (buf->strides[i] != buf->itemsize) {
PyErr_SetString(PyExc_ValueError,
"Buffer and memoryview are not contiguous in the same dimension.");
goto fail;
}
}
if (spec & (__Pyx_MEMVIEW_STRIDED | __Pyx_MEMVIEW_FOLLOW)) {
if (buf->strides[i] < buf->itemsize) {
PyErr_SetString(PyExc_ValueError,
"Buffer and memoryview are not contiguous in the same dimension.");
goto fail;
}
}
if (spec & __Pyx_MEMVIEW_DIRECT) {
if (buf->suboffsets && buf->suboffsets[i] >= 0) {
PyErr_SetString(PyExc_ValueError,
"Buffer not compatible with direct access.");
goto fail;
}
}
if (spec & (__Pyx_MEMVIEW_PTR | __Pyx_MEMVIEW_FULL)) {
if (!buf->suboffsets) {
PyErr_SetString(PyExc_ValueError,
"Buffer not able to be indirectly accessed.");
goto fail;
}
}
if (spec & __Pyx_MEMVIEW_PTR) {
if (buf->suboffsets[i] < 0) {
PyErr_Format(PyExc_ValueError,
"Buffer not indirectly accessed in %d dimension, although memoryview is.", i);
goto fail;
}
}
}
if (c_or_f_flag & __Pyx_IS_F_CONTIG) {
stride = 1;
for(i=0; i<ndim; i++) {
if(stride * buf->itemsize != buf->strides[i]) {
PyErr_SetString(PyExc_ValueError,
"Buffer not fortran contiguous.");
goto fail;
}
stride = stride * buf->shape[i];
}
} else if (c_or_f_flag & __Pyx_IS_F_CONTIG) {
for(i=ndim-1; i>-1; i--) {
if(stride * buf->itemsize != buf->strides[i]) {
PyErr_SetString(PyExc_ValueError,
"Buffer not C contiguous.");
goto fail;
}
stride = stride * buf->shape[i];
}
}
if(unlikely(__Pyx_init_memviewslice(memview, ndim, memviewslice) == -1)) {
goto fail;
}
retval = 0;
goto no_fail;
fail:
__Pyx_XDECREF(memviewslice->memview);
memviewslice->memview = 0;
memviewslice->data = 0;
retval = -1;
no_fail:
__Pyx_RefNannyFinishContext();
return retval;
}
static int __Pyx_init_memviewslice(
struct __pyx_memoryview_obj *memview,
int ndim,
__Pyx_memviewslice *memviewslice) {
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("init_memviewslice");
int i, retval=-1;
Py_buffer *buf = &memview->view;
if(!buf) {
PyErr_SetString(PyExc_ValueError,
"buf is NULL.");
goto fail;
} else if(memviewslice->memview || memviewslice->data) {
PyErr_SetString(PyExc_ValueError,
"memviewslice is already initialized!");
goto fail;
}
for(i=0; i<ndim; i++) {
memviewslice->strides[i] = buf->strides[i];
memviewslice->shape[i] = buf->shape[i];
if(buf->suboffsets) {
memviewslice->suboffsets[i] = buf->suboffsets[i];
} else {
memviewslice->suboffsets[i] = -1;
}
}
__Pyx_INCREF((PyObject *)memview);
__Pyx_GIVEREF((PyObject *)memview);
memviewslice->memview = memview;
memviewslice->data = (char *)buf->buf;
retval = 0;
goto no_fail;
fail:
__Pyx_XDECREF(memviewslice->memview);
memviewslice->memview = 0;
memviewslice->data = 0;
retval = -1;
no_fail:
__Pyx_RefNannyFinishContext();
return retval;
}
// UtilityCode: MemviewSliceCopyTemplate
static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
__Pyx_RefNannyDeclarations
int i;
__Pyx_memviewslice new_mvs = {0, 0};
struct __pyx_memoryview_obj *from_memview = from_mvs.memview;
Py_buffer *buf = &from_memview->view;
PyObject *shape_tuple = 0;
PyObject *temp_int = 0;
struct __pyx_array_obj *array_obj = 0;
struct __pyx_memoryview_obj *memview_obj = 0;
char *mode = "%(mode)s";
__Pyx_RefNannySetupContext("%(copy_name)s");
shape_tuple = PyTuple_New((Py_ssize_t)(buf->ndim));
if(unlikely(!shape_tuple)) {
goto fail;
}
__Pyx_GOTREF(shape_tuple);
for(i=0; i<buf->ndim; i++) {
temp_int = PyInt_FromLong(buf->shape[i]);
if(unlikely(!temp_int)) {
goto fail;
} else {
PyTuple_SET_ITEM(shape_tuple, i, temp_int);
}
}
array_obj = __pyx_array_new(shape_tuple, %(sizeof_dtype)s, buf->format, mode);
if (unlikely(!array_obj)) {
goto fail;
}
__Pyx_GOTREF(array_obj);
memview_obj = __pyx_memoryview_new((PyObject *) array_obj, %(contig_flag)s);
if (unlikely(!memview_obj)) {
goto fail;