Commit 3f8fefb0 authored by Mark Florisson's avatar Mark Florisson

Tempita support

parent 2cd11fd8
...@@ -12,6 +12,7 @@ import os ...@@ -12,6 +12,7 @@ import os
import re import re
import codecs import codecs
import glob
import Naming import Naming
import Options import Options
import StringEncoding import StringEncoding
...@@ -20,6 +21,7 @@ from Scanning import SourceDescriptor ...@@ -20,6 +21,7 @@ from Scanning import SourceDescriptor
from Cython.StringIOTree import StringIOTree from Cython.StringIOTree import StringIOTree
import DebugFlags import DebugFlags
import Errors import Errors
from Cython import Tempita as tempita
from Cython.Utils import none_or_sub from Cython.Utils import none_or_sub
try: try:
...@@ -42,6 +44,9 @@ uncachable_builtins = [ ...@@ -42,6 +44,9 @@ uncachable_builtins = [
'WindowsError', 'WindowsError',
] ]
Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
Utility_dir = os.path.join(Cython_dir, "Utility")
class UtilityCodeBase(object): class UtilityCodeBase(object):
is_cython_utility = False is_cython_utility = False
...@@ -69,8 +74,7 @@ class UtilityCodeBase(object): ...@@ -69,8 +74,7 @@ class UtilityCodeBase(object):
if utilities: if utilities:
return utilities return utilities
Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) filename = os.path.join(Utility_dir, path)
filename = os.path.join(Cython_dir, "Utility", path)
f = codecs.open(filename, encoding='UTF-8') f = codecs.open(filename, encoding='UTF-8')
_, ext = os.path.splitext(path) _, ext = os.path.splitext(path)
...@@ -111,8 +115,8 @@ class UtilityCodeBase(object): ...@@ -111,8 +115,8 @@ class UtilityCodeBase(object):
return utilities return utilities
@classmethod @classmethod
def load_utility_from_file(cls, path, util_code_name, proto_fmt_dict=None, def load_utility_from_file(cls, path, util_code_name,
impl_fmt_dict=None, *args, **kwargs): context=None, **kwargs):
""" """
Load a utility code from a file specified by path (relative to Load a utility code from a file specified by path (relative to
Cython/Utility) and name util_code_name. Cython/Utility) and name util_code_name.
...@@ -125,37 +129,52 @@ class UtilityCodeBase(object): ...@@ -125,37 +129,52 @@ class UtilityCodeBase(object):
for prototypes and implementation respectively. For non-python or for prototypes and implementation respectively. For non-python or
-cython files a // comment should be used instead. -cython files a // comment should be used instead.
proto_fmt_dict and impl_format_dict can optionally be given to perform If context is given, the utility is considered a tempita template.
any substitutions. The context dict (which may be empty) will be unpacked to form
all the variables in the template.
If the @cname decorator is not used and this is a CythonUtilityCode, If the @cname decorator is not used and this is a CythonUtilityCode,
one should pass in the 'name' keyword argument to be used for name one should pass in the 'name' keyword argument to be used for name
mangling of such entries. mangling of such entries.
""" """
proto, impl = cls.load_utility_as_string(path, util_code_name) proto, impl = cls.load_utility_as_string(path, util_code_name, context)
if proto: if proto is not None:
if proto_fmt_dict:
proto = proto % proto_fmt_dict
kwargs['proto'] = proto kwargs['proto'] = proto
if impl is not None:
if impl:
if impl_fmt_dict:
impl = impl % impl_fmt_dict
kwargs['impl'] = impl kwargs['impl'] = impl
if 'name' not in kwargs: if 'name' not in kwargs:
kwargs['name'] = os.path.splitext(path)[0] kwargs['name'] = os.path.splitext(path)[0]
return cls(*args, **kwargs) return cls(**kwargs)
@classmethod @classmethod
def load_utility_as_string(cls, path, util_code_name): def load_utility_as_string(cls, path, util_code_name, context=None):
""" """
Load a utility code as a string. Returns (proto, implementation) Load a utility code as a string. Returns (proto, implementation)
""" """
utilities = cls.load_utilities_from_file(path) utilities = cls.load_utilities_from_file(path)
return utilities[util_code_name]
proto, impl = utilities[util_code_name]
if proto:
if context is not None:
proto = tempita.sub(proto, **context)
if impl:
if context is not None:
impl = tempita.sub(impl, **context)
return proto, impl
@classmethod
def load_utility(cls, name, context=None, **kwargs):
"Load utility name with context from a utility file name.suffix"
files = glob.glob(os.path.join(Utility_dir, name + '.*'))
if len(files) != 1:
raise ValueError("Need exactly one utility file")
return cls.load_utilities_from_file(files[0], name, context, **kwargs)
def __str__(self): def __str__(self):
return "<%s(%s)" % (type(self).__name__, self.name) return "<%s(%s)" % (type(self).__name__, self.name)
...@@ -1275,6 +1294,12 @@ class CCodeWriter(object): ...@@ -1275,6 +1294,12 @@ class CCodeWriter(object):
elif fix_indent: elif fix_indent:
self.level += 1 self.level += 1
def putln_tempita(self, code, **context):
self.putln(tempita.sub(code, **context))
def put_tempita(self, code, **context):
self.put(tempita.sub(code, **context))
def increase_indent(self): def increase_indent(self):
self.level = self.level + 1 self.level = self.level + 1
......
...@@ -2345,8 +2345,12 @@ class IndexNode(ExprNode): ...@@ -2345,8 +2345,12 @@ class IndexNode(ExprNode):
skip_child_analysis = False skip_child_analysis = False
buffer_access = False buffer_access = False
# memoryviewslice_access = False memoryviewslice_access = False
if self.base.type.is_buffer or self.base.type.is_memoryviewslice:
if (self.base.type.is_memoryviewslice and not self.indices and
isinstance(self.index, EllipsisNode)):
memoryviewslice_access = True
elif self.base.type.is_buffer or self.base.type.is_memoryviewslice:
if self.indices: if self.indices:
indices = self.indices indices = self.indices
else: else:
...@@ -2390,11 +2394,11 @@ class IndexNode(ExprNode): ...@@ -2390,11 +2394,11 @@ class IndexNode(ExprNode):
if self.type.is_buffer: if self.type.is_buffer:
self.base.entry.buffer_aux.writable_needed = True self.base.entry.buffer_aux.writable_needed = True
# elif memoryviewslice_access: elif memoryviewslice_access:
# self.type = self.base.type self.type = self.base.type
# self.is_memoryviewslice_access = True self.is_memoryviewslice_access = True
# if getting: if getting:
# error(self.pos, "memoryviews currently support setting only.") error(self.pos, "memoryviews currently support setting only.")
else: else:
base_type = self.base.type base_type = self.base.type
......
...@@ -112,7 +112,7 @@ def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code ...@@ -112,7 +112,7 @@ def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code
for i in range(ndim): for i in range(ndim):
code.putln("%s.shape[%d] = %s.shape[%d];" % (lhs_cname, i, rhs_cname, i)) code.putln("%s.shape[%d] = %s.shape[%d];" % (lhs_cname, i, rhs_cname, i))
code.putln("%s.strides[%d] = %s.strides[%d];" % (lhs_cname, i, rhs_cname, i)) code.putln("%s.strides[%d] = %s.strides[%d];" % (lhs_cname, i, rhs_cname, i))
# code.putln("%s.suboffsets[%d] = %s.suboffsets[%d];" % (lhs_cname, i, rhs_cname, i)) code.putln("%s.suboffsets[%d] = %s.suboffsets[%d];" % (lhs_cname, i, rhs_cname, i))
def get_buf_flag(specs): def get_buf_flag(specs):
is_c_contig, is_f_contig = is_cf_contig(specs) is_c_contig, is_f_contig = is_cf_contig(specs)
...@@ -186,7 +186,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): ...@@ -186,7 +186,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
self.buf_ptr_type = dtype self.buf_ptr_type = dtype
def get_buf_suboffsetvars(self): def get_buf_suboffsetvars(self):
return self._for_all_ndim("%s.memview->view.suboffsets[%d]") return self._for_all_ndim("%s.suboffsets[%d]")
def get_buf_stridevars(self): def get_buf_stridevars(self):
return self._for_all_ndim("%s.strides[%d]") return self._for_all_ndim("%s.strides[%d]")
...@@ -267,7 +267,7 @@ static int %s(const __Pyx_memviewslice mvs) { ...@@ -267,7 +267,7 @@ static int %s(const __Pyx_memviewslice mvs) {
%(for_loop)s { %(for_loop)s {
#ifdef DEBUG #ifdef DEBUG
/* printf("mvs.suboffsets[i] %%d\\n", mvs.suboffsets[i]); */ printf("mvs.suboffsets[i] %%d\\n", mvs.suboffsets[i]);
printf("mvs.strides[i] %%d\\n", mvs.strides[i]); printf("mvs.strides[i] %%d\\n", mvs.strides[i]);
printf("mvs.shape[i] %%d\\n", mvs.shape[i]); printf("mvs.shape[i] %%d\\n", mvs.shape[i]);
printf("size %%d\\n", size); printf("size %%d\\n", size);
...@@ -275,7 +275,7 @@ static int %s(const __Pyx_memviewslice mvs) { ...@@ -275,7 +275,7 @@ static int %s(const __Pyx_memviewslice mvs) {
#endif #endif
#undef DEBUG #undef DEBUG
if(mvs.memview->view.suboffsets[i] >= 0) { if(mvs.suboffsets[i] >= 0) {
return 0; return 0;
} }
if(size * itemsize != mvs.strides[i]) { if(size * itemsize != mvs.strides[i]) {
...@@ -374,13 +374,17 @@ class CopyFuncUtilCode(object): ...@@ -374,13 +374,17 @@ class CopyFuncUtilCode(object):
mode = 'fortran' mode = 'fortran'
contig_flag = "PyBUF_F_CONTIGUOUS" contig_flag = "PyBUF_F_CONTIGUOUS"
code.put(copy_template % context = dict(
dict(
copy_name=self.copy_func_name, copy_name=self.copy_func_name,
mode=mode, mode=mode,
sizeof_dtype="sizeof(%s)" % self.from_memview.dtype.declaration_code(''), sizeof_dtype="sizeof(%s)" % self.from_memview.dtype.declaration_code(''),
contig_flag=contig_flag, contig_flag=contig_flag,
copy_contents_name=copy_contents_name)) copy_contents_name=copy_contents_name
)
_, copy_code = UtilityCode.load_utility_as_string(
"MemoryView_C.c", "MemviewSliceCopyTemplate", context)
code.put(copy_code)
def get_copy_contents_func(from_mvs, to_mvs, cfunc_name): def get_copy_contents_func(from_mvs, to_mvs, cfunc_name):
...@@ -744,10 +748,8 @@ def load_memview_c_utility(name, *args, **kwargs): ...@@ -744,10 +748,8 @@ def load_memview_c_utility(name, *args, **kwargs):
def load_memview_c_string(name): def load_memview_c_string(name):
return UtilityCode.load_utility_as_string("MemoryView_C.c", name) return UtilityCode.load_utility_as_string("MemoryView_C.c", name)
_, copy_template = UtilityCode.load_utility_as_string(
"MemoryView_C.c", "MemviewSliceCopyTemplate")
fmt_dict = { context = {
'memview_struct_name': memview_objstruct_cname, 'memview_struct_name': memview_objstruct_cname,
'max_dims': Options.buffer_max_dims, 'max_dims': Options.buffer_max_dims,
'memviewslice_name': memviewslice_cname, 'memviewslice_name': memviewslice_cname,
...@@ -755,10 +757,10 @@ fmt_dict = { ...@@ -755,10 +757,10 @@ fmt_dict = {
memviewslice_declare_code = load_memview_c_utility( memviewslice_declare_code = load_memview_c_utility(
name="MemviewSliceStruct", name="MemviewSliceStruct",
proto_block='utility_code_proto_before_types', proto_block='utility_code_proto_before_types',
proto_fmt_dict=fmt_dict) context=context)
memviewslice_init_code = load_memview_c_utility( memviewslice_init_code = load_memview_c_utility(
name="MemviewSliceInit", name="MemviewSliceInit",
proto_fmt_dict={'BUF_MAX_NDIMS': Options.buffer_max_dims}, context={'BUF_MAX_NDIMS': Options.buffer_max_dims},
requires=[memviewslice_declare_code], requires=[memviewslice_declare_code],
) )
This diff is collapsed.
"""
Helper for looping over sequences, particular in templates.
Often in a loop in a template it's handy to know what's next up,
previously up, if this is the first or last item in the sequence, etc.
These can be awkward to manage in a normal Python loop, but using the
looper you can get a better sense of the context. Use like::
>>> for loop, item in looper(['a', 'b', 'c']):
... print loop.number, item
... if not loop.last:
... print '---'
1 a
---
2 b
---
3 c
"""
import sys
from Cython.Tempita.compat3 import basestring_
__all__ = ['looper']
class looper(object):
"""
Helper for looping (particularly in templates)
Use this like::
for loop, item in looper(seq):
if loop.first:
...
"""
def __init__(self, seq):
self.seq = seq
def __iter__(self):
return looper_iter(self.seq)
def __repr__(self):
return '<%s for %r>' % (
self.__class__.__name__, self.seq)
class looper_iter(object):
def __init__(self, seq):
self.seq = list(seq)
self.pos = 0
def __iter__(self):
return self
def __next__(self):
if self.pos >= len(self.seq):
raise StopIteration
result = loop_pos(self.seq, self.pos), self.seq[self.pos]
self.pos += 1
return result
if sys.version < "3":
next = __next__
class loop_pos(object):
def __init__(self, seq, pos):
self.seq = seq
self.pos = pos
def __repr__(self):
return '<loop pos=%r at %r>' % (
self.seq[self.pos], self.pos)
def index(self):
return self.pos
index = property(index)
def number(self):
return self.pos + 1
number = property(number)
def item(self):
return self.seq[self.pos]
item = property(item)
def __next__(self):
try:
return self.seq[self.pos + 1]
except IndexError:
return None
__next__ = property(__next__)
if sys.version < "3":
next = __next__
def previous(self):
if self.pos == 0:
return None
return self.seq[self.pos - 1]
previous = property(previous)
def odd(self):
return not self.pos % 2
odd = property(odd)
def even(self):
return self.pos % 2
even = property(even)
def first(self):
return self.pos == 0
first = property(first)
def last(self):
return self.pos == len(self.seq) - 1
last = property(last)
def length(self):
return len(self.seq)
length = property(length)
def first_group(self, getter=None):
"""
Returns true if this item is the start of a new group,
where groups mean that some attribute has changed. The getter
can be None (the item itself changes), an attribute name like
``'.attr'``, a function, or a dict key or list index.
"""
if self.first:
return True
return self._compare_group(self.item, self.previous, getter)
def last_group(self, getter=None):
"""
Returns true if this item is the end of a new group,
where groups mean that some attribute has changed. The getter
can be None (the item itself changes), an attribute name like
``'.attr'``, a function, or a dict key or list index.
"""
if self.last:
return True
return self._compare_group(self.item, self.__next__, getter)
def _compare_group(self, item, other, getter):
if getter is None:
return item != other
elif (isinstance(getter, basestring_)
and getter.startswith('.')):
getter = getter[1:]
if getter.endswith('()'):
getter = getter[:-2]
return getattr(item, getter)() != getattr(other, getter)()
else:
return getattr(item, getter) != getattr(other, getter)
elif hasattr(getter, '__call__'):
return getter(item) != getter(other)
else:
return item[getter] != other[getter]
import sys
__all__ = ['b', 'basestring_', 'bytes', 'next', 'is_unicode']
if sys.version < "3":
b = bytes = str
basestring_ = basestring
else:
def b(s):
if isinstance(s, str):
return s.encode('latin1')
return bytes(s)
basestring_ = (bytes, str)
bytes = bytes
text = str
if sys.version < "3":
def next(obj):
return obj.next()
else:
next = next
if sys.version < "3":
def is_unicode(obj):
return isinstance(obj, unicode)
else:
def is_unicode(obj):
return isinstance(obj, str)
def coerce_text(v):
if not isinstance(v, basestring_):
if sys.version < "3":
attr = '__unicode__'
else:
attr = '__str__'
if hasattr(v, attr):
return unicode(v)
else:
return bytes(v)
return v
...@@ -3,13 +3,13 @@ ...@@ -3,13 +3,13 @@
/* memoryview slice struct */ /* memoryview slice struct */
typedef struct { typedef struct {
struct %(memview_struct_name)s *memview; struct {{memview_struct_name}} *memview;
/* For convenience and faster access */ /* For convenience and faster access */
char *data; char *data;
Py_ssize_t shape[%(max_dims)d]; Py_ssize_t shape[{{max_dims}}];
Py_ssize_t strides[%(max_dims)d]; Py_ssize_t strides[{{max_dims}}];
/* Py_ssize_t suboffsets[%(max_dims)d]; */ Py_ssize_t suboffsets[{{max_dims}}];
} %(memviewslice_name)s; } {{memviewslice_name}};
// UtilityProto: MemviewSliceInit // UtilityProto: MemviewSliceInit
...@@ -194,12 +194,11 @@ static int __Pyx_init_memviewslice( ...@@ -194,12 +194,11 @@ static int __Pyx_init_memviewslice(
for(i=0; i<ndim; i++) { for(i=0; i<ndim; i++) {
memviewslice->strides[i] = buf->strides[i]; memviewslice->strides[i] = buf->strides[i];
memviewslice->shape[i] = buf->shape[i]; memviewslice->shape[i] = buf->shape[i];
/*
if(buf->suboffsets) { if(buf->suboffsets) {
memviewslice->suboffsets[i] = buf->suboffsets[i]; memviewslice->suboffsets[i] = buf->suboffsets[i];
} else { } else {
memviewslice->suboffsets[i] = -1; memviewslice->suboffsets[i] = -1;
}*/ }
} }
__Pyx_INCREF((PyObject *)memview); __Pyx_INCREF((PyObject *)memview);
...@@ -221,7 +220,7 @@ no_fail: ...@@ -221,7 +220,7 @@ no_fail:
// UtilityCode: MemviewSliceCopyTemplate // UtilityCode: MemviewSliceCopyTemplate
static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) { static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
__Pyx_RefNannyDeclarations __Pyx_RefNannyDeclarations
int i; int i;
...@@ -232,9 +231,9 @@ static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) { ...@@ -232,9 +231,9 @@ static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
PyObject *temp_int = 0; PyObject *temp_int = 0;
struct __pyx_array_obj *array_obj = 0; struct __pyx_array_obj *array_obj = 0;
struct __pyx_memoryview_obj *memview_obj = 0; struct __pyx_memoryview_obj *memview_obj = 0;
char *mode = "%(mode)s"; char *mode = "{{mode}}";
__Pyx_RefNannySetupContext("%(copy_name)s"); __Pyx_RefNannySetupContext("{{copy_name}}");
shape_tuple = PyTuple_New((Py_ssize_t)(buf->ndim)); shape_tuple = PyTuple_New((Py_ssize_t)(buf->ndim));
if(unlikely(!shape_tuple)) { if(unlikely(!shape_tuple)) {
...@@ -252,13 +251,13 @@ static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) { ...@@ -252,13 +251,13 @@ static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
} }
} }
array_obj = __pyx_array_new(shape_tuple, %(sizeof_dtype)s, buf->format, mode); array_obj = __pyx_array_new(shape_tuple, {{sizeof_dtype}}, buf->format, mode);
if (unlikely(!array_obj)) { if (unlikely(!array_obj)) {
goto fail; goto fail;
} }
__Pyx_GOTREF(array_obj); __Pyx_GOTREF(array_obj);
memview_obj = __pyx_memoryview_new((PyObject *) array_obj, %(contig_flag)s); memview_obj = __pyx_memoryview_new((PyObject *) array_obj, {{contig_flag}});
if (unlikely(!memview_obj)) { if (unlikely(!memview_obj)) {
goto fail; goto fail;
} }
...@@ -270,7 +269,7 @@ static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) { ...@@ -270,7 +269,7 @@ static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
goto fail; goto fail;
} }
if (unlikely(-1 == %(copy_contents_name)s(&from_mvs, &new_mvs))) { if (unlikely(-1 == {{copy_contents_name}}(&from_mvs, &new_mvs))) {
/* PyErr_SetString(PyExc_RuntimeError, /* PyErr_SetString(PyExc_RuntimeError,
"Could not copy contents of memoryview slice."); */ "Could not copy contents of memoryview slice."); */
goto fail; goto fail;
......
...@@ -274,6 +274,7 @@ packages = [ ...@@ -274,6 +274,7 @@ packages = [
'Cython.Build.Tests', 'Cython.Build.Tests',
'Cython.Compiler.Tests', 'Cython.Compiler.Tests',
'Cython.Utility', 'Cython.Utility',
'Cython.Tempita',
] ]
if include_debugger: if include_debugger:
......
...@@ -20,7 +20,10 @@ def create_array(shape, mode='c'): ...@@ -20,7 +20,10 @@ def create_array(shape, mode='c'):
def slice_contig_indexing(): def slice_contig_indexing():
""" """
>>> slice_contig_indexing() >>> print "disabled"
disabled
slice_contig_indexing()
98 98
61 61
98 98
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment