Skip to content

C
cython
Commit 556722cb authored by Mark Florisson's avatar Mark Florisson
Browse files
Options

Utility Code loader + memview python3 compat

parent c8be949c
Hide whitespace changes
Inline Side-by-side
Showing with 711 additions and 543 deletions
Cython/Compiler/Buffer.py
View file @ 556722cb
......@@ -1090,9 +1090,6 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
}
static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) {
if (!ts) {
PyErr_SetString(PyExc_ValueError, "Got NULL buffer format");
}
int got_Z = 0;
while (1) {
switch(*ts) {
......
Cython/Compiler/Code.pxd
View file @ 556722cb
cimport cython
cdef class UtilityCode:
cdef class UtilityCodeBase(object):
pass
cdef class UtilityCode(UtilityCodeBase):
cdef public object proto
cdef public object impl
cdef public object init
......
Cython/Compiler/Code.py
View file @ 556722cb
......@@ -8,7 +8,10 @@ cython.declare(re=object, Naming=object, Options=object, StringEncoding=object,
Utils=object, SourceDescriptor=object, StringIOTree=object,
DebugFlags=object, none_or_sub=object, basestring=object)
import os
import re
import codecs
import Naming
import Options
import StringEncoding
......@@ -16,6 +19,7 @@ from Cython import Utils
from Scanning import SourceDescriptor
from Cython.StringIOTree import StringIOTree
import DebugFlags
import Errors
from Cython.Utils import none_or_sub
try:
......@@ -38,17 +42,130 @@ uncachable_builtins = [
'WindowsError',
]
class UtilityCode(object):
class UtilityCodeBase(object):
is_cython_utility = False
_utility_cache = {}
@classmethod
def _add_utility(self, utility, type, lines, begin_lineno):
if utility:
code = '\n' * begin_lineno + ''.join(lines)
if type == 'Proto':
utility[0] = code
else:
utility[1] = code
@classmethod
def load_utilities_from_file(cls, path):
utilities = cls._utility_cache.get(path)
if utilities:
return utilities
Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
filename = os.path.join(Cython_dir, "Utility", path)
f = codecs.open(filename, encoding='UTF-8')
_, ext = os.path.splitext(path)
if ext in ('.pyx', '.py', '.pxd', '.pxi'):
comment = '#'
else:
comment = '//'
regex = r'%s\s*Utility(Proto|Code)\s*:\s*((\w|\.)+)\s*' % comment
flags = re.DOTALL
utilities = {}
lines = []
utility = type = None
begin_lineno = 0
for lineno, line in enumerate(f):
m = re.search(regex, line)
if m:
cls._add_utility(utility, type, lines, begin_lineno)
begin_lineno = lineno + 1
type, name = m.group(1), m.group(2)
utility = utilities.setdefault(name, [None, None])
utilities[name] = utility
lines = []
else:
lines.append(line)
if not utility:
raise ValueError("Empty utility code file")
# Don't forget to add the last utility code
cls._add_utility(utility, type, lines, begin_lineno)
f.close()
cls._utility_cache[path] = utilities
return utilities
@classmethod
def load_utility_from_file(cls, path, util_code_name, proto_fmt_dict=None,
impl_fmt_dict=None, *args, **kwargs):
"""
Load a utility code from a file specified by path (relative to
Cython/Utility) and name util_code_name.
Utilities in the file can be specified as follows:
# UtilityProto: name
# UtilityImpl: name
for prototypes and implementation respectively. For non-python or
-cython files a // comment should be used instead.
proto_fmt_dict and impl_format_dict can optionally be given to perform
any substitutions.
If the @cname decorator is not used and this is a CythonUtilityCode,
one should pass in the 'name' keyword argument to be used for name
mangling of such entries.
"""
proto, impl = cls.load_utility_as_string(path, util_code_name)
if proto:
if proto_fmt_dict:
proto = proto % proto_fmt_dict
kwargs['proto'] = proto
if impl:
if impl_fmt_dict:
impl = impl % impl_fmt_dict
kwargs['impl'] = impl
if 'name' not in kwargs:
kwargs['name'] = os.path.splitext(path)[0]
return cls(*args, **kwargs)
@classmethod
def load_utility_as_string(cls, path, util_code_name):
"""
Load a utility code as a string. Returns (proto, implementation)
"""
utilities = cls.load_utilities_from_file(path)
return utilities[util_code_name]
def __str__(self):
return "<%s(%s)" % (type(self).__name__, self.name)
class UtilityCode(UtilityCodeBase):
# Stores utility code to add during code generation.
#
# See GlobalState.put_utility_code.
#
# hashes/equals by instance
is_cython_utility = False
def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
proto_block='utility_code_proto'):
proto_block='utility_code_proto', name=None):
# proto_block: Which code block to dump prototype in. See GlobalState.
self.proto = proto
self.impl = impl
......@@ -58,6 +175,7 @@ class UtilityCode(object):
self._cache = {}
self.specialize_list = []
self.proto_block = proto_block
self.name = name
def get_tree(self):
pass
......
Cython/Compiler/CythonScope.py
View file @ 556722cb
......@@ -5,6 +5,7 @@ from Errors import error
from Scanning import StringSourceDescriptor
import Options
import Buffer
import MemoryView
class CythonScope(ModuleScope):
is_cython_builtin = 1
......@@ -100,11 +101,12 @@ def create_cython_scope(context, create_testscope):
return scope
cython_testscope_utility_code = CythonUtilityCode(u"""
@cname('__pyx_testscope')
cdef object _testscope(int value):
return "hello from cython scope, value=%d" % value
""")
# Load test utilities for the cython scope
def load_testscope_utility(cython_util_name, *args, **kwargs):
return CythonUtilityCode.load_utility_from_file(
"TestCythonScope.pyx", cython_util_name, *args, **kwargs)
undecorated_methods_protos = UtilityCode(proto=u"""
/* These methods are undecorated and have therefore no prototype */
......@@ -116,237 +118,18 @@ undecorated_methods_protos = UtilityCode(proto=u"""
PyObject *self, PyObject *value);
""")
test_cython_utility_dep = CythonUtilityCode(u"""
@cname('__pyx_test_dep')
cdef test_dep(obj):
print 'test_dep', obj
""")
cython_test_extclass_utility_code = CythonUtilityCode(
name="TestClassUtilityCode",
prefix="__pyx_prefix_TestClass_",
requires=[undecorated_methods_protos, test_cython_utility_dep],
impl=u"""
cdef extern from *:
cdef object __pyx_test_dep(object)
@cname('__pyx_TestClass')
cdef class TestClass(object):
cdef public int value
def __init__(self, int value):
self.value = value
def __str__(self):
return 'TestClass(%d)' % self.value
cdef cdef_method(self, int value):
print 'Hello from cdef_method', value
cpdef cpdef_method(self, int value):
print 'Hello from cpdef_method', value
def def_method(self, int value):
print 'Hello from def_method', value
@cname('cdef_cname')
cdef cdef_cname_method(self, int value):
print "Hello from cdef_cname_method", value
@cname('cpdef_cname')
cpdef cpdef_cname_method(self, int value):
print "Hello from cpdef_cname_method", value
@cname('def_cname')
def def_cname_method(self, int value):
print "Hello from def_cname_method", value
@cname('__pyx_test_call_other_cy_util')
cdef test_call(obj):
print 'test_call'
__pyx_test_dep(obj)
@cname('__pyx_TestClass_New')
cdef _testclass_new(int value):
return TestClass(value)
""")
cythonview_testscope_utility_code = CythonUtilityCode(u"""
@cname('__pyx_view_testscope')
cdef object _testscope(int value):
return "hello from cython.view scope, value=%d" % value
""")
memview_name = u'memoryview'
memview_typeptr_cname = '__pyx_memoryview_type'
memview_objstruct_cname = '__pyx_memoryview_obj'
view_utility_code = CythonUtilityCode(
u"""
cdef class Enum(object):
cdef object name
def __init__(self, name):
self.name = name
def __repr__(self):
return self.name
cdef strided = Enum("<strided axis packing mode>")
cdef contig = Enum("<contig axis packing mode>")
cdef follow = Enum("<follow axis packing mode>")
cdef direct = Enum("<direct axis access mode>")
cdef ptr = Enum("<ptr axis access mode>")
cdef full = Enum("<full axis access mode>")
cdef extern from *:
int __Pyx_GetBuffer(object, Py_buffer *, int)
void __Pyx_ReleaseBuffer(Py_buffer *)
@cname('__pyx_memoryview')
cdef class memoryview(object):
cdef object obj
cdef Py_buffer view
cdef int gotbuf_flag
def __cinit__(memoryview self, object obj, int flags):
self.obj = obj
__Pyx_GetBuffer(self.obj, &self.view, flags)
def __dealloc__(memoryview self):
self.obj = None
__Pyx_ReleaseBuffer(&self.view)
@cname('__pyx_memoryview_new')
cdef memoryview memoryview_cwrapper(object o, int flags):
return memoryview(o, flags)
""", name="view_code", requires=(Buffer.GetAndReleaseBufferUtilityCode(),))
cython_array_utility_code = CythonUtilityCode(u'''
cdef extern from "stdlib.h":
void *malloc(size_t)
void free(void *)
cdef extern from "Python.h":
cdef enum:
PyBUF_C_CONTIGUOUS,
PyBUF_F_CONTIGUOUS,
PyBUF_ANY_CONTIGUOUS
@cname("__pyx_array")
cdef class array:
cdef:
char *data
Py_ssize_t len
char *format
int ndim
Py_ssize_t *shape
Py_ssize_t *strides
Py_ssize_t itemsize
str mode
void (*callback_free_data)(char *data)
def __cinit__(array self, tuple shape, Py_ssize_t itemsize, char *format,
mode="c", bint allocate_buffer=True):
self.ndim = len(shape)
self.itemsize = itemsize
if not self.ndim:
raise ValueError("Empty shape tuple for cython.array")
if self.itemsize <= 0:
raise ValueError("itemsize <= 0 for cython.array")
self.format = format
self.shape = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
self.strides = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
if not self.shape or not self.strides:
raise MemoryError("unable to allocate shape or strides.")
cdef int idx
# cdef Py_ssize_t dim, stride
idx = 0
for dim in shape:
if dim <= 0:
raise ValueError("Invalid shape.")
self.shape[idx] = dim
idx += 1
stride = itemsize
if mode == "fortran":
idx = 0
for dim in shape:
self.strides[idx] = stride
stride = stride * dim
idx += 1
elif mode == "c":
idx = self.ndim-1
for dim in shape[::-1]:
self.strides[idx] = stride
stride = stride * dim
idx -= 1
else:
raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode)
self.len = stride
self.mode = mode
if allocate_buffer:
self.data = <char *>malloc(self.len)
if not self.data:
raise MemoryError("unable to allocate array data.")
else:
self.data = NULL
self.callback_free_data = NULL
cython_testscope_utility_code = load_testscope_utility("TestScope")
def __getbuffer__(self, Py_buffer *info, int flags):
test_cython_utility_dep = load_testscope_utility("TestDep")
cdef int bufmode = -1
if self.mode == b"c":
bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
elif self.mode == b"fortran":
bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
if not (flags & bufmode):
raise ValueError("Can only create a buffer that is contiguous in memory.")
info.buf = self.data
info.len = self.len
info.ndim = self.ndim
info.shape = self.shape
info.strides = self.strides
info.suboffsets = NULL
info.itemsize = self.itemsize
info.format = self.format
# we do not need to call releasebuffer
info.obj = None
cython_test_extclass_utility_code = \
load_testscope_utility("TestClass", name="TestClass",
requires=[undecorated_methods_protos,
test_cython_utility_dep])
def __releasebuffer__(array self, Py_buffer* info):
# array.__releasebuffer__ should not be called,
# because the Py_buffer's 'obj' field is set to None.
raise NotImplementedError()
cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
def __dealloc__(array self):
if self.data:
if self.callback_free_data != NULL:
self.callback_free_data(self.data)
else:
free(self.data)
self.data = NULL
if self.strides:
free(self.strides)
self.strides = NULL
if self.shape:
free(self.shape)
self.shape = NULL
self.format = NULL
self.itemsize = 0
view_utility_code = MemoryView.load_memview_cy_utility(
"MemoryView", requires=(Buffer.GetAndReleaseBufferUtilityCode(),))
@cname("__pyx_array_new")
cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode):
return array(shape, itemsize, format, mode)
''')
cython_array_utility_code = MemoryView.load_memview_cy_utility("CythonArray")
Cython/Compiler/MemoryView.py
View file @ 556722cb
......@@ -2,7 +2,6 @@ from Errors import CompileError
from ExprNodes import IntNode, NoneNode, IntBinopNode, NameNode, AttributeNode
from Visitor import CythonTransform
import Options
import CythonScope
from Code import UtilityCode
from UtilityCode import CythonUtilityCode
from PyrexTypes import py_object_type, cython_memoryview_ptr_type
......@@ -47,6 +46,10 @@ _spec_to_const = {
'full' : MEMVIEW_FULL
}
memview_name = u'memoryview'
memview_typeptr_cname = '__pyx_memoryview_type'
memview_objstruct_cname = '__pyx_memoryview_obj'
memviewslice_cname = u'__Pyx_memviewslice'
def specs_to_code(specs):
arr = []
......@@ -694,309 +697,33 @@ class MemoryViewSliceTransform(CythonTransform):
def visit_SingleAssignmentNode(self, node):
return node
copy_template = '''
static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
__Pyx_RefNannyDeclarations
int i;
__Pyx_memviewslice new_mvs = {0, 0};
struct __pyx_memoryview_obj *from_memview = from_mvs.memview;
Py_buffer *buf = &from_memview->view;
PyObject *shape_tuple = 0;
PyObject *temp_int = 0;
struct __pyx_array_obj *array_obj = 0;
struct __pyx_memoryview_obj *memview_obj = 0;
char *mode = "%(mode)s";
__Pyx_RefNannySetupContext("%(copy_name)s");
shape_tuple = PyTuple_New((Py_ssize_t)(buf->ndim));
if(unlikely(!shape_tuple)) {
goto fail;
}
__Pyx_GOTREF(shape_tuple);
for(i=0; i<buf->ndim; i++) {
temp_int = PyInt_FromLong(buf->shape[i]);
if(unlikely(!temp_int)) {
goto fail;
} else {
PyTuple_SET_ITEM(shape_tuple, i, temp_int);
}
}
array_obj = __pyx_array_new(shape_tuple, %(sizeof_dtype)s, buf->format, mode);
if (unlikely(!array_obj)) {
goto fail;
}
__Pyx_GOTREF(array_obj);
memview_obj = __pyx_memoryview_new((PyObject *) array_obj, %(contig_flag)s);
if (unlikely(!memview_obj)) {
goto fail;
}
def load_memview_cy_utility(name, *args, **kwargs):
return CythonUtilityCode.load_utility_from_file(
"MemoryView.pyx", name, *args, **kwargs)
/* initialize new_mvs */
if (unlikely(-1 == __Pyx_init_memviewslice(memview_obj, buf->ndim, &new_mvs))) {
PyErr_SetString(PyExc_RuntimeError,
"Could not initialize new memoryviewslice object.");
goto fail;
}
def load_memview_c_utility(name, *args, **kwargs):
return UtilityCode.load_utility_from_file(
"MemoryView.c", name, *args, **kwargs)
if (unlikely(-1 == %(copy_contents_name)s(&from_mvs, &new_mvs))) {
/* PyErr_SetString(PyExc_RuntimeError,
"Could not copy contents of memoryview slice."); */
goto fail;
}
def load_memview_c_string(name):
return UtilityCode.load_utility_as_string("MemoryView.c", name)
goto no_fail;
fail:
__Pyx_XDECREF(new_mvs.memview); new_mvs.memview = 0;
new_mvs.data = 0;
no_fail:
__Pyx_XDECREF(shape_tuple); shape_tuple = 0;
__Pyx_GOTREF(temp_int);
__Pyx_XDECREF(temp_int); temp_int = 0;
__Pyx_XDECREF(array_obj); array_obj = 0;
__Pyx_RefNannyFinishContext();
return new_mvs;
_, copy_template = UtilityCode.load_utility_as_string(
"MemoryView.c", "MemviewSliceCopyTemplate")
fmt_dict = {
'memview_struct_name': memview_objstruct_cname,
'max_dims': Options.buffer_max_dims,
'memviewslice_name': memviewslice_cname,
}
'''
spec_constants_code = UtilityCode(proto="""
#define __Pyx_MEMVIEW_DIRECT 1
#define __Pyx_MEMVIEW_PTR 2
#define __Pyx_MEMVIEW_FULL 4
#define __Pyx_MEMVIEW_CONTIG 8
#define __Pyx_MEMVIEW_STRIDED 16
#define __Pyx_MEMVIEW_FOLLOW 32
"""
)
memviewslice_cname = u'__Pyx_memviewslice'
memviewslice_declare_code = UtilityCode(
memviewslice_declare_code = load_memview_c_utility(
name="MemviewSliceStruct",
proto_block='utility_code_proto_before_types',
proto="""
/* memoryview slice struct */
proto_fmt_dict=fmt_dict)
typedef struct {
struct %(memview_struct_name)s *memview;
char *data;
Py_ssize_t shape[%(max_dims)d];
Py_ssize_t strides[%(max_dims)d];
Py_ssize_t suboffsets[%(max_dims)d];
} %(memviewslice_name)s;
""" % dict(memview_struct_name=CythonScope.memview_objstruct_cname,
max_dims=Options.buffer_max_dims,
memviewslice_name=memviewslice_cname)
memviewslice_init_code = load_memview_c_utility(
name="MemviewSliceInit",
proto_fmt_dict={'BUF_MAX_NDIMS': Options.buffer_max_dims},
requires=[memviewslice_declare_code],
)
memviewslice_init_code = UtilityCode(proto="""\
#define __Pyx_BUF_MAX_NDIMS %(BUF_MAX_NDIMS)d
#define __Pyx_MEMVIEW_DIRECT 1
#define __Pyx_MEMVIEW_PTR 2
#define __Pyx_MEMVIEW_FULL 4
#define __Pyx_MEMVIEW_CONTIG 8
#define __Pyx_MEMVIEW_STRIDED 16
#define __Pyx_MEMVIEW_FOLLOW 32
#define __Pyx_IS_C_CONTIG 1
#define __Pyx_IS_F_CONTIG 2
static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview,
int *axes_specs, int c_or_f_flag, int ndim, __Pyx_TypeInfo *dtype,
__Pyx_BufFmt_StackElem stack[], __Pyx_memviewslice *memviewslice);
static int __Pyx_init_memviewslice(
struct __pyx_memoryview_obj *memview,
int ndim,
__Pyx_memviewslice *memviewslice);
""" % {'BUF_MAX_NDIMS' :Options.buffer_max_dims},
impl = """\
static int __Pyx_ValidateAndInit_memviewslice(
struct __pyx_memoryview_obj *memview,
int *axes_specs,
int c_or_f_flag,
int ndim,
__Pyx_TypeInfo *dtype,
__Pyx_BufFmt_StackElem stack[],
__Pyx_memviewslice *memviewslice) {
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("ValidateAndInit_memviewslice");
Py_buffer *buf = &memview->view;
int stride, i, spec = 0, retval = -1;
if (!buf) goto fail;
if(memviewslice->data || memviewslice->memview) {
PyErr_SetString(PyExc_ValueError,
"memoryviewslice struct must be initialized to NULL.");
goto fail;
}
if (buf->ndim != ndim) {
PyErr_Format(PyExc_ValueError,
"Buffer has wrong number of dimensions (expected %d, got %d)",
ndim, buf->ndim);
goto fail;
}
__Pyx_BufFmt_Context ctx;
__Pyx_BufFmt_Init(&ctx, stack, dtype);
if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail;
if ((unsigned)buf->itemsize != dtype->size) {
PyErr_Format(PyExc_ValueError,
"Item size of buffer (%"PY_FORMAT_SIZE_T"d byte%s) does not match size of '%s' (%"PY_FORMAT_SIZE_T"d byte%s)",
buf->itemsize, (buf->itemsize > 1) ? "s" : "",
dtype->name,
dtype->size, (dtype->size > 1) ? "s" : "");
goto fail;
}
if (!buf->strides) {
PyErr_SetString(PyExc_ValueError,
"buffer does not supply strides necessary for memoryview.");
goto fail;
}
for(i=0; i<ndim; i++) {
spec = axes_specs[i];
if (spec & __Pyx_MEMVIEW_CONTIG) {
if (buf->strides[i] != buf->itemsize) {
PyErr_SetString(PyExc_ValueError,
"Buffer and memoryview are not contiguous in the same dimension.");
goto fail;
}
}
if (spec & (__Pyx_MEMVIEW_STRIDED | __Pyx_MEMVIEW_FOLLOW)) {
if (buf->strides[i] < buf->itemsize) {
PyErr_SetString(PyExc_ValueError,
"Buffer and memoryview are not contiguous in the same dimension.");
goto fail;
}
}
if (spec & __Pyx_MEMVIEW_DIRECT) {
if (buf->suboffsets && buf->suboffsets[i] >= 0) {
PyErr_SetString(PyExc_ValueError,
"Buffer not compatible with direct access.");
goto fail;
}
}
if (spec & (__Pyx_MEMVIEW_PTR | __Pyx_MEMVIEW_FULL)) {
if (!buf->suboffsets) {
PyErr_SetString(PyExc_ValueError,
"Buffer not able to be indirectly accessed.");
goto fail;
}
}
if (spec & __Pyx_MEMVIEW_PTR) {
if (buf->suboffsets[i] < 0) {
PyErr_Format(PyExc_ValueError,
"Buffer not indirectly accessed in %d dimension, although memoryview is.", i);
goto fail;
}
}
}
if (c_or_f_flag & __Pyx_IS_F_CONTIG) {
stride = 1;
for(i=0; i<ndim; i++) {
if(stride * buf->itemsize != buf->strides[i]) {
PyErr_SetString(PyExc_ValueError,
"Buffer not fortran contiguous.");
goto fail;
}
stride = stride * buf->shape[i];
}
} else if (c_or_f_flag & __Pyx_IS_F_CONTIG) {
for(i=ndim-1; i>-1; i--) {
if(stride * buf->itemsize != buf->strides[i]) {
PyErr_SetString(PyExc_ValueError,
"Buffer not C contiguous.");
goto fail;
}
stride = stride * buf->shape[i];
}
}
if(unlikely(__Pyx_init_memviewslice(memview, ndim, memviewslice) == -1)) {
goto fail;
}
retval = 0;
goto no_fail;
fail:
__Pyx_XDECREF(memviewslice->memview);
memviewslice->memview = 0;
memviewslice->data = 0;
retval = -1;
no_fail:
__Pyx_RefNannyFinishContext();
return retval;
}
static int __Pyx_init_memviewslice(
struct __pyx_memoryview_obj *memview,
int ndim,
__Pyx_memviewslice *memviewslice) {
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("init_memviewslice");
int i, retval=-1;
Py_buffer *buf = &memview->view;
if(!buf) {
PyErr_SetString(PyExc_ValueError,
"buf is NULL.");
goto fail;
} else if(memviewslice->memview || memviewslice->data) {
PyErr_SetString(PyExc_ValueError,
"memviewslice is already initialized!");
goto fail;
}
for(i=0; i<ndim; i++) {
memviewslice->strides[i] = buf->strides[i];
memviewslice->shape[i] = buf->shape[i];
if(buf->suboffsets) {
memviewslice->suboffsets[i] = buf->suboffsets[i];
} else {
memviewslice->suboffsets[i] = -1;
}
}
__Pyx_INCREF((PyObject *)memview);
__Pyx_GIVEREF((PyObject *)memview);
memviewslice->memview = memview;
memviewslice->data = (char *)buf->buf;
retval = 0;
goto no_fail;
fail:
__Pyx_XDECREF(memviewslice->memview);
memviewslice->memview = 0;
memviewslice->data = 0;
retval = -1;
no_fail:
__Pyx_RefNannyFinishContext();
return retval;
}
""")
memviewslice_init_code.requires = [memviewslice_declare_code]
Cython/Compiler/PyrexTypes.py
View file @ 556722cb
......@@ -498,6 +498,7 @@ class MemoryViewSliceType(PyrexType):
def check_for_null_code(self, cname):
return cname + '.memview'
class BufferType(BaseType):
#
# Delegates most attribute
......
Cython/Compiler/UtilityCode.py
View file @ 556722cb
......@@ -3,6 +3,7 @@ from Scanning import StringSourceDescriptor
import Symtab
import Naming
from Cython.Compiler import Visitor
import Code
class NonManglingModuleScope(Symtab.ModuleScope):
......@@ -43,7 +44,7 @@ class CythonUtilityCodeContext(StringParseContext):
return self.scope
class CythonUtilityCode(object):
class CythonUtilityCode(Code.UtilityCodeBase):
"""
Utility code written in the Cython language itself.
......
Cython/Utility/MemoryView.c 0 → 100644
View file @ 556722cb
// UtilityProto: MemviewSliceStruct
/* memoryview slice struct */
typedef struct {
struct %(memview_struct_name)s *memview;
char *data;
Py_ssize_t shape[%(max_dims)d];
Py_ssize_t strides[%(max_dims)d];
Py_ssize_t suboffsets[%(max_dims)d];
} %(memviewslice_name)s;
// UtilityProto: MemviewSliceInit
#define __Pyx_BUF_MAX_NDIMS %(BUF_MAX_NDIMS)d
#define __Pyx_MEMVIEW_DIRECT 1
#define __Pyx_MEMVIEW_PTR 2
#define __Pyx_MEMVIEW_FULL 4
#define __Pyx_MEMVIEW_CONTIG 8
#define __Pyx_MEMVIEW_STRIDED 16
#define __Pyx_MEMVIEW_FOLLOW 32
#define __Pyx_IS_C_CONTIG 1
#define __Pyx_IS_F_CONTIG 2
static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview,
int *axes_specs, int c_or_f_flag, int ndim, __Pyx_TypeInfo *dtype,
__Pyx_BufFmt_StackElem stack[], __Pyx_memviewslice *memviewslice);
static int __Pyx_init_memviewslice(
struct __pyx_memoryview_obj *memview,
int ndim,
__Pyx_memviewslice *memviewslice);
// UtilityCode: MemviewSliceInit
static int __Pyx_ValidateAndInit_memviewslice(
struct __pyx_memoryview_obj *memview,
int *axes_specs,
int c_or_f_flag,
int ndim,
__Pyx_TypeInfo *dtype,
__Pyx_BufFmt_StackElem stack[],
__Pyx_memviewslice *memviewslice) {
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("ValidateAndInit_memviewslice");
Py_buffer *buf = &memview->view;
int stride, i, spec = 0, retval = -1;
if (!buf) goto fail;
if(memviewslice->data || memviewslice->memview) {
PyErr_SetString(PyExc_ValueError,
"memoryviewslice struct must be initialized to NULL.");
goto fail;
}
if (buf->ndim != ndim) {
PyErr_Format(PyExc_ValueError,
"Buffer has wrong number of dimensions (expected %d, got %d)",
ndim, buf->ndim);
goto fail;
}
__Pyx_BufFmt_Context ctx;
__Pyx_BufFmt_Init(&ctx, stack, dtype);
if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail;
if ((unsigned)buf->itemsize != dtype->size) {
PyErr_Format(PyExc_ValueError,
"Item size of buffer (%" PY_FORMAT_SIZE_T "d byte%s) "
"does not match size of '%s' (%" PY_FORMAT_SIZE_T "d byte%s)",
buf->itemsize,
(buf->itemsize > 1) ? "s" : "",
dtype->name,
dtype->size,
(dtype->size > 1) ? "s" : "");
goto fail;
}
if (!buf->strides) {
PyErr_SetString(PyExc_ValueError,
"buffer does not supply strides necessary for memoryview.");
goto fail;
}
for(i=0; i<ndim; i++) {
spec = axes_specs[i];
if (spec & __Pyx_MEMVIEW_CONTIG) {
if (buf->strides[i] != buf->itemsize) {
PyErr_SetString(PyExc_ValueError,
"Buffer and memoryview are not contiguous in the same dimension.");
goto fail;
}
}
if (spec & (__Pyx_MEMVIEW_STRIDED | __Pyx_MEMVIEW_FOLLOW)) {
if (buf->strides[i] < buf->itemsize) {
PyErr_SetString(PyExc_ValueError,
"Buffer and memoryview are not contiguous in the same dimension.");
goto fail;
}
}
if (spec & __Pyx_MEMVIEW_DIRECT) {
if (buf->suboffsets && buf->suboffsets[i] >= 0) {
PyErr_SetString(PyExc_ValueError,
"Buffer not compatible with direct access.");
goto fail;
}
}
if (spec & (__Pyx_MEMVIEW_PTR | __Pyx_MEMVIEW_FULL)) {
if (!buf->suboffsets) {
PyErr_SetString(PyExc_ValueError,
"Buffer not able to be indirectly accessed.");
goto fail;
}
}
if (spec & __Pyx_MEMVIEW_PTR) {
if (buf->suboffsets[i] < 0) {
PyErr_Format(PyExc_ValueError,
"Buffer not indirectly accessed in %d dimension, although memoryview is.", i);
goto fail;
}
}
}
if (c_or_f_flag & __Pyx_IS_F_CONTIG) {
stride = 1;
for(i=0; i<ndim; i++) {
if(stride * buf->itemsize != buf->strides[i]) {
PyErr_SetString(PyExc_ValueError,
"Buffer not fortran contiguous.");
goto fail;
}
stride = stride * buf->shape[i];
}
} else if (c_or_f_flag & __Pyx_IS_F_CONTIG) {
for(i=ndim-1; i>-1; i--) {
if(stride * buf->itemsize != buf->strides[i]) {
PyErr_SetString(PyExc_ValueError,
"Buffer not C contiguous.");
goto fail;
}
stride = stride * buf->shape[i];
}
}
if(unlikely(__Pyx_init_memviewslice(memview, ndim, memviewslice) == -1)) {
goto fail;
}
retval = 0;
goto no_fail;
fail:
__Pyx_XDECREF(memviewslice->memview);
memviewslice->memview = 0;
memviewslice->data = 0;
retval = -1;
no_fail:
__Pyx_RefNannyFinishContext();
return retval;
}
static int __Pyx_init_memviewslice(
struct __pyx_memoryview_obj *memview,
int ndim,
__Pyx_memviewslice *memviewslice) {
__Pyx_RefNannyDeclarations
__Pyx_RefNannySetupContext("init_memviewslice");
int i, retval=-1;
Py_buffer *buf = &memview->view;
if(!buf) {
PyErr_SetString(PyExc_ValueError,
"buf is NULL.");
goto fail;
} else if(memviewslice->memview || memviewslice->data) {
PyErr_SetString(PyExc_ValueError,
"memviewslice is already initialized!");
goto fail;
}
for(i=0; i<ndim; i++) {
memviewslice->strides[i] = buf->strides[i];
memviewslice->shape[i] = buf->shape[i];
if(buf->suboffsets) {
memviewslice->suboffsets[i] = buf->suboffsets[i];
} else {
memviewslice->suboffsets[i] = -1;
}
}
__Pyx_INCREF((PyObject *)memview);
__Pyx_GIVEREF((PyObject *)memview);
memviewslice->memview = memview;
memviewslice->data = (char *)buf->buf;
retval = 0;
goto no_fail;
fail:
__Pyx_XDECREF(memviewslice->memview);
memviewslice->memview = 0;
memviewslice->data = 0;
retval = -1;
no_fail:
__Pyx_RefNannyFinishContext();
return retval;
}
// UtilityCode: MemviewSliceCopyTemplate
static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
__Pyx_RefNannyDeclarations
int i;
__Pyx_memviewslice new_mvs = {0, 0};
struct __pyx_memoryview_obj *from_memview = from_mvs.memview;
Py_buffer *buf = &from_memview->view;
PyObject *shape_tuple = 0;
PyObject *temp_int = 0;
struct __pyx_array_obj *array_obj = 0;
struct __pyx_memoryview_obj *memview_obj = 0;
char *mode = "%(mode)s";
__Pyx_RefNannySetupContext("%(copy_name)s");
shape_tuple = PyTuple_New((Py_ssize_t)(buf->ndim));
if(unlikely(!shape_tuple)) {
goto fail;
}
__Pyx_GOTREF(shape_tuple);
for(i=0; i<buf->ndim; i++) {
temp_int = PyInt_FromLong(buf->shape[i]);
if(unlikely(!temp_int)) {
goto fail;
} else {
PyTuple_SET_ITEM(shape_tuple, i, temp_int);
}
}
array_obj = __pyx_array_new(shape_tuple, %(sizeof_dtype)s, buf->format, mode);
if (unlikely(!array_obj)) {
goto fail;
}
__Pyx_GOTREF(array_obj);
memview_obj = __pyx_memoryview_new((PyObject *) array_obj, %(contig_flag)s);
if (unlikely(!memview_obj)) {
goto fail;
}
/* initialize new_mvs */
if (unlikely(-1 == __Pyx_init_memviewslice(memview_obj, buf->ndim, &new_mvs))) {
PyErr_SetString(PyExc_RuntimeError,
"Could not initialize new memoryviewslice object.");
goto fail;
}
if (unlikely(-1 == %(copy_contents_name)s(&from_mvs, &new_mvs))) {
/* PyErr_SetString(PyExc_RuntimeError,
"Could not copy contents of memoryview slice."); */
goto fail;
}
goto no_fail;
fail:
__Pyx_XDECREF(new_mvs.memview); new_mvs.memview = 0;
new_mvs.data = 0;
no_fail:
__Pyx_XDECREF(shape_tuple); shape_tuple = 0;
__Pyx_GOTREF(temp_int);
__Pyx_XDECREF(temp_int); temp_int = 0;
__Pyx_XDECREF(array_obj); array_obj = 0;
__Pyx_RefNannyFinishContext();
return new_mvs;
}
Cython/Utility/MemoryView.pyx 0 → 100644
View file @ 556722cb
# UtilityCode: CythonArray
cdef extern from "stdlib.h":
void *malloc(size_t)
void free(void *)
cdef extern from "Python.h":
cdef enum:
PyBUF_C_CONTIGUOUS,
PyBUF_F_CONTIGUOUS,
PyBUF_ANY_CONTIGUOUS
@cname("__pyx_array")
cdef class array:
cdef:
char *data
Py_ssize_t len
char *format
int ndim
Py_ssize_t *shape
Py_ssize_t *strides
Py_ssize_t itemsize
unicode mode
bytes _format
void (*callback_free_data)(char *data)
def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format,
mode=u"c", bint allocate_buffer=True):
self.ndim = len(shape)
self.itemsize = itemsize
if not self.ndim:
raise ValueError("Empty shape tuple for cython.array")
if self.itemsize <= 0:
raise ValueError("itemsize <= 0 for cython.array")
encode = getattr(format, 'encode', None)
if encode:
format = encode('ASCII')
self._format = format
self.format = self._format
self.shape = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
self.strides = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
if not self.shape or not self.strides:
raise MemoryError("unable to allocate shape or strides.")
cdef int idx
# cdef Py_ssize_t dim, stride
idx = 0
for dim in shape:
if dim <= 0:
raise ValueError("Invalid shape.")
self.shape[idx] = dim
idx += 1
stride = itemsize
if mode == "fortran":
idx = 0
for dim in shape:
self.strides[idx] = stride
stride = stride * dim
idx += 1
elif mode == "c":
idx = self.ndim-1
for dim in shape[::-1]:
self.strides[idx] = stride
stride = stride * dim
idx -= 1
else:
raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode)
self.len = stride
decode = getattr(mode, 'decode', None)
if decode:
mode = decode('ASCII')
self.mode = mode
if allocate_buffer:
self.data = <char *>malloc(self.len)
if not self.data:
raise MemoryError("unable to allocate array data.")
else:
self.data = NULL
self.callback_free_data = NULL
def __getbuffer__(self, Py_buffer *info, int flags):
cdef int bufmode = -1
if self.mode == b"c":
bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
elif self.mode == b"fortran":
bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
if not (flags & bufmode):
raise ValueError("Can only create a buffer that is contiguous in memory.")
info.buf = self.data
info.len = self.len
info.ndim = self.ndim
info.shape = self.shape
info.strides = self.strides
info.suboffsets = NULL
info.itemsize = self.itemsize
info.format = self.format
# we do not need to call releasebuffer
info.obj = None
def __releasebuffer__(array self, Py_buffer* info):
# array.__releasebuffer__ should not be called,
# because the Py_buffer's 'obj' field is set to None.
raise NotImplementedError()
def __dealloc__(array self):
if self.data:
if self.callback_free_data != NULL:
self.callback_free_data(self.data)
else:
free(self.data)
self.data = NULL
if self.strides:
free(self.strides)
self.strides = NULL
if self.shape:
free(self.shape)
self.shape = NULL
self.format = NULL
self.itemsize = 0
@cname("__pyx_array_new")
cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode):
return array(shape, itemsize, format, mode.decode('ASCII'))
# UtilityCode: MemoryView
@cname('__pyx_MemviewEnum')
cdef class Enum(object):
cdef object name
def __init__(self, name):
self.name = name
def __repr__(self):
return self.name
cdef strided = Enum("<strided axis packing mode>")
cdef contig = Enum("<contig axis packing mode>")
cdef follow = Enum("<follow axis packing mode>")
cdef direct = Enum("<direct axis access mode>")
cdef ptr = Enum("<ptr axis access mode>")
cdef full = Enum("<full axis access mode>")
cdef extern from *:
int __Pyx_GetBuffer(object, Py_buffer *, int)
void __Pyx_ReleaseBuffer(Py_buffer *)
@cname('__pyx_memoryview')
cdef class memoryview(object):
cdef object obj
cdef Py_buffer view
cdef int gotbuf_flag
def __cinit__(memoryview self, object obj, int flags):
self.obj = obj
__Pyx_GetBuffer(self.obj, &self.view, flags)
def __dealloc__(memoryview self):
self.obj = None
__Pyx_ReleaseBuffer(&self.view)
@cname('__pyx_memoryview_new')
cdef memoryview memoryview_cwrapper(object o, int flags):
return memoryview(o, flags)
Cython/Utility/TestCythonScope.pyx 0 → 100644
View file @ 556722cb
# UtilityCode: TestClass
# These utilities are for testing purposes
cdef extern from *:
cdef object __pyx_test_dep(object)
@cname('__pyx_TestClass')
cdef class TestClass(object):
cdef public int value
def __init__(self, int value):
self.value = value
def __str__(self):
return 'TestClass(%d)' % self.value
cdef cdef_method(self, int value):
print 'Hello from cdef_method', value
cpdef cpdef_method(self, int value):
print 'Hello from cpdef_method', value
def def_method(self, int value):
print 'Hello from def_method', value
@cname('cdef_cname')
cdef cdef_cname_method(self, int value):
print "Hello from cdef_cname_method", value
@cname('cpdef_cname')
cpdef cpdef_cname_method(self, int value):
print "Hello from cpdef_cname_method", value
@cname('def_cname')
def def_cname_method(self, int value):
print "Hello from def_cname_method", value
@cname('__pyx_test_call_other_cy_util')
cdef test_call(obj):
print 'test_call'
__pyx_test_dep(obj)
@cname('__pyx_TestClass_New')
cdef _testclass_new(int value):
return TestClass(value)
# UtilityCode: TestDep
@cname('__pyx_test_dep')
cdef test_dep(obj):
print 'test_dep', obj
# UtilityCode: TestScope
@cname('__pyx_testscope')
cdef object _testscope(int value):
return "hello from cython scope, value=%d" % value
# UtilityCode: View.TestScope
@cname('__pyx_view_testscope')
cdef object _testscope(int value):
return "hello from cython.view scope, value=%d" % value
MANIFEST.in
View file @ 556722cb
......@@ -17,7 +17,8 @@ include Demos/embed/*
include Demos/freeze/*
include Demos/libraries/*
include Demos/Makefile*
recursive-include Cython/Debugger/Tests *
recursive-include Cython/Debugger/Tests *.pyx *.pxd *.c *.h
recursive-include Cython/Utility *
recursive-include Tools *
recursive-include tests *.pyx *.pxd *.pxi *.py *.h *.BROKEN bugs.txt
recursive-include tests *_lib.cpp *.srctree
......
runtests.py
View file @ 556722cb
......@@ -1187,6 +1187,7 @@ def refactor_for_py3(distdir, cy3_dir):
recursive-exclude Cython *
recursive-include Cython *.py *.pyx *.pxd
recursive-include Cython/Debugger/Tests *
recursive-include Cython/Utility *
include runtests.py
include cython.py
''')
......
setup.py
View file @ 556722cb
......@@ -63,6 +63,7 @@ else:
'Cython.Plex' : ['*.pxd'],
'Cython.Compiler' : ['*.pxd'],
'Cython.Runtime' : ['*.pyx', '*.pxd'],
'Cython.Utility' : ['*.pyx', '*.pxd', '*.c', '*.h'],
'Cython' : [ p[7:] for p in pxd_include_patterns ],
}
......@@ -272,6 +273,7 @@ packages = [
'Cython.Tests',
'Cython.Build.Tests',
'Cython.Compiler.Tests',
'Cython.Utility',
]
if include_debugger:
......@@ -280,7 +282,6 @@ if include_debugger:
# it's enough to do this for Py2.5+:
setup_args['package_data']['Cython.Debugger.Tests'] = ['codefile', 'cfuncs.c']
setup(
name = 'Cython',
version = version,
......
tests/run/cythonarray.pyx
View file @ 556722cb
# mode: run
from __future__ import unicode_literals
# from cython cimport array
# cimport cython.array as array
cimport cython as cy
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7