Tempita support

Cython/Compiler/Code.py

@@ -12,6 +12,7 @@ import os
 import re
 import codecs
 
+import glob
 import Naming
 import Options
 import StringEncoding
@@ -20,6 +21,7 @@ from Scanning import SourceDescriptor
 from Cython.StringIOTree import StringIOTree
 import DebugFlags
 import Errors
+from Cython import Tempita as tempita
 
 from Cython.Utils import none_or_sub
 try:
@@ -42,6 +44,9 @@ uncachable_builtins = [
     'WindowsError',
     ]
 
+Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+Utility_dir = os.path.join(Cython_dir, "Utility")
+
 class UtilityCodeBase(object):
 
     is_cython_utility = False
@@ -69,8 +74,7 @@ class UtilityCodeBase(object):
         if utilities:
             return utilities
 
-        Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-        filename = os.path.join(Cython_dir, "Utility", path)
+        filename = os.path.join(Utility_dir, path)
         f = codecs.open(filename, encoding='UTF-8')
 
         _, ext = os.path.splitext(path)
@@ -111,8 +115,8 @@ class UtilityCodeBase(object):
         return utilities
 
     @classmethod
-    def load_utility_from_file(cls, path, util_code_name, proto_fmt_dict=None,
-                               impl_fmt_dict=None, *args, **kwargs):
+    def load_utility_from_file(cls, path, util_code_name,
+                               context=None, **kwargs):
         """
         Load a utility code from a file specified by path (relative to
         Cython/Utility) and name util_code_name.
@@ -125,37 +129,52 @@ class UtilityCodeBase(object):
         for prototypes and implementation respectively. For non-python or
         -cython files a // comment should be used instead.
 
-        proto_fmt_dict and impl_format_dict can optionally be given to perform
-        any substitutions.
+        If context is given, the utility is considered a tempita template.
+        The context dict (which may be empty) will be unpacked to form
+        all the variables in the template.
 
         If the @cname decorator is not used and this is a CythonUtilityCode,
         one should pass in the 'name' keyword argument to be used for name
         mangling of such entries.
         """
-        proto, impl = cls.load_utility_as_string(path, util_code_name)
+        proto, impl = cls.load_utility_as_string(path, util_code_name, context)
 
-        if proto:
-            if proto_fmt_dict:
-                proto = proto % proto_fmt_dict
+        if proto is not None:
             kwargs['proto'] = proto
-
-        if impl:
-            if impl_fmt_dict:
-                impl = impl % impl_fmt_dict
+        if impl is not None:
             kwargs['impl'] = impl
 
         if 'name' not in kwargs:
             kwargs['name'] = os.path.splitext(path)[0]
 
-        return cls(*args, **kwargs)
+        return cls(**kwargs)
 
     @classmethod
-    def load_utility_as_string(cls, path, util_code_name):
+    def load_utility_as_string(cls, path, util_code_name, context=None):
         """
         Load a utility code as a string. Returns (proto, implementation)
         """
         utilities = cls.load_utilities_from_file(path)
-        return utilities[util_code_name]
+
+        proto, impl = utilities[util_code_name]
+        if proto:
+            if context is not None:
+                proto = tempita.sub(proto, **context)
+
+        if impl:
+            if context is not None:
+                impl = tempita.sub(impl, **context)
+
+        return proto, impl
+
+    @classmethod
+    def load_utility(cls, name, context=None, **kwargs):
+        "Load utility name with context from a utility file name.suffix"
+        files = glob.glob(os.path.join(Utility_dir, name + '.*'))
+        if len(files) != 1:
+            raise ValueError("Need exactly one utility file")
+
+        return cls.load_utilities_from_file(files[0], name, context, **kwargs)
 
     def __str__(self):
         return "<%s(%s)" % (type(self).__name__, self.name)
@@ -1275,6 +1294,12 @@ class CCodeWriter(object):
         elif fix_indent:
             self.level += 1
 
+    def putln_tempita(self, code, **context):
+        self.putln(tempita.sub(code, **context))
+
+    def put_tempita(self, code, **context):
+        self.put(tempita.sub(code, **context))
+
     def increase_indent(self):
         self.level = self.level + 1
 

Cython/Compiler/ExprNodes.py

@@ -2345,8 +2345,12 @@ class IndexNode(ExprNode):
 
         skip_child_analysis = False
         buffer_access = False
-#        memoryviewslice_access = False
-        if self.base.type.is_buffer or self.base.type.is_memoryviewslice:
+        memoryviewslice_access = False
+
+        if (self.base.type.is_memoryviewslice and not self.indices and
+                isinstance(self.index, EllipsisNode)):
+            memoryviewslice_access = True
+        elif self.base.type.is_buffer or self.base.type.is_memoryviewslice:
             if self.indices:
                 indices = self.indices
             else:
@@ -2390,11 +2394,11 @@ class IndexNode(ExprNode):
                     if self.type.is_buffer:
                         self.base.entry.buffer_aux.writable_needed = True
 
-#        elif memoryviewslice_access:
-#            self.type = self.base.type
-#            self.is_memoryviewslice_access = True
-#            if getting:
-#                error(self.pos, "memoryviews currently support setting only.")
+        elif memoryviewslice_access:
+            self.type = self.base.type
+            self.is_memoryviewslice_access = True
+            if getting:
+                error(self.pos, "memoryviews currently support setting only.")
 
         else:
             base_type = self.base.type

Cython/Compiler/MemoryView.py

@@ -112,7 +112,7 @@ def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code
     for i in range(ndim):
         code.putln("%s.shape[%d] = %s.shape[%d];" % (lhs_cname, i, rhs_cname, i))
         code.putln("%s.strides[%d] = %s.strides[%d];" % (lhs_cname, i, rhs_cname, i))
-        # code.putln("%s.suboffsets[%d] = %s.suboffsets[%d];" % (lhs_cname, i, rhs_cname, i))
+        code.putln("%s.suboffsets[%d] = %s.suboffsets[%d];" % (lhs_cname, i, rhs_cname, i))
 
 def get_buf_flag(specs):
     is_c_contig, is_f_contig = is_cf_contig(specs)
@@ -186,7 +186,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
         self.buf_ptr_type = dtype
 
     def get_buf_suboffsetvars(self):
-        return self._for_all_ndim("%s.memview->view.suboffsets[%d]")
+        return self._for_all_ndim("%s.suboffsets[%d]")
 
     def get_buf_stridevars(self):
         return self._for_all_ndim("%s.strides[%d]")
@@ -267,7 +267,7 @@ static int %s(const __Pyx_memviewslice mvs) {
     %(for_loop)s {
 
 #ifdef DEBUG
-        /* printf("mvs.suboffsets[i] %%d\\n", mvs.suboffsets[i]); */
+        printf("mvs.suboffsets[i] %%d\\n", mvs.suboffsets[i]);
         printf("mvs.strides[i] %%d\\n", mvs.strides[i]);
         printf("mvs.shape[i] %%d\\n", mvs.shape[i]);
         printf("size %%d\\n", size);
@@ -275,7 +275,7 @@ static int %s(const __Pyx_memviewslice mvs) {
 #endif
 #undef DEBUG
 
-        if(mvs.memview->view.suboffsets[i] >= 0) {
+        if(mvs.suboffsets[i] >= 0) {
             return 0;
         }
         if(size * itemsize != mvs.strides[i]) {
@@ -374,13 +374,17 @@ class CopyFuncUtilCode(object):
             mode = 'fortran'
             contig_flag = "PyBUF_F_CONTIGUOUS"
 
-        code.put(copy_template %
-            dict(
+        context = dict(
             copy_name=self.copy_func_name,
             mode=mode,
             sizeof_dtype="sizeof(%s)" % self.from_memview.dtype.declaration_code(''),
             contig_flag=contig_flag,
-                 copy_contents_name=copy_contents_name))
+            copy_contents_name=copy_contents_name
+        )
+
+        _, copy_code = UtilityCode.load_utility_as_string(
+                    "MemoryView_C.c", "MemviewSliceCopyTemplate", context)
+        code.put(copy_code)
 
 
 def get_copy_contents_func(from_mvs, to_mvs, cfunc_name):
@@ -744,10 +748,8 @@ def load_memview_c_utility(name, *args, **kwargs):
 def load_memview_c_string(name):
     return UtilityCode.load_utility_as_string("MemoryView_C.c", name)
 
-_, copy_template = UtilityCode.load_utility_as_string(
-                "MemoryView_C.c", "MemviewSliceCopyTemplate")
 
-fmt_dict = {
+context = {
     'memview_struct_name': memview_objstruct_cname,
     'max_dims': Options.buffer_max_dims,
     'memviewslice_name': memviewslice_cname,
@@ -755,10 +757,10 @@ fmt_dict = {
 memviewslice_declare_code = load_memview_c_utility(
         name="MemviewSliceStruct",
         proto_block='utility_code_proto_before_types',
-        proto_fmt_dict=fmt_dict)
+        context=context)
 
 memviewslice_init_code = load_memview_c_utility(
     name="MemviewSliceInit",
-    proto_fmt_dict={'BUF_MAX_NDIMS': Options.buffer_max_dims},
+    context={'BUF_MAX_NDIMS': Options.buffer_max_dims},
     requires=[memviewslice_declare_code],
 )

Cython/Tempita/_looper.py

+"""
+Helper for looping over sequences, particular in templates.
+
+Often in a loop in a template it's handy to know what's next up,
+previously up, if this is the first or last item in the sequence, etc.
+These can be awkward to manage in a normal Python loop, but using the
+looper you can get a better sense of the context.  Use like::
+
+    >>> for loop, item in looper(['a', 'b', 'c']):
+    ...     print loop.number, item
+    ...     if not loop.last:
+    ...         print '---'
+    1 a
+    ---
+    2 b
+    ---
+    3 c
+
+"""
+
+import sys
+from Cython.Tempita.compat3 import basestring_
+
+__all__ = ['looper']
+
+
+class looper(object):
+    """
+    Helper for looping (particularly in templates)
+
+    Use this like::
+
+        for loop, item in looper(seq):
+            if loop.first:
+                ...
+    """
+
+    def __init__(self, seq):
+        self.seq = seq
+
+    def __iter__(self):
+        return looper_iter(self.seq)
+
+    def __repr__(self):
+        return '<%s for %r>' % (
+            self.__class__.__name__, self.seq)
+
+
+class looper_iter(object):
+
+    def __init__(self, seq):
+        self.seq = list(seq)
+        self.pos = 0
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self.pos >= len(self.seq):
+            raise StopIteration
+        result = loop_pos(self.seq, self.pos), self.seq[self.pos]
+        self.pos += 1
+        return result
+
+    if sys.version < "3":
+        next = __next__
+
+
+class loop_pos(object):
+
+    def __init__(self, seq, pos):
+        self.seq = seq
+        self.pos = pos
+
+    def __repr__(self):
+        return '<loop pos=%r at %r>' % (
+            self.seq[self.pos], self.pos)
+
+    def index(self):
+        return self.pos
+    index = property(index)
+
+    def number(self):
+        return self.pos + 1
+    number = property(number)
+
+    def item(self):
+        return self.seq[self.pos]
+    item = property(item)
+
+    def __next__(self):
+        try:
+            return self.seq[self.pos + 1]
+        except IndexError:
+            return None
+    __next__ = property(__next__)
+
+    if sys.version < "3":
+        next = __next__
+
+    def previous(self):
+        if self.pos == 0:
+            return None
+        return self.seq[self.pos - 1]
+    previous = property(previous)
+
+    def odd(self):
+        return not self.pos % 2
+    odd = property(odd)
+
+    def even(self):
+        return self.pos % 2
+    even = property(even)
+
+    def first(self):
+        return self.pos == 0
+    first = property(first)
+
+    def last(self):
+        return self.pos == len(self.seq) - 1
+    last = property(last)
+
+    def length(self):
+        return len(self.seq)
+    length = property(length)
+
+    def first_group(self, getter=None):
+        """
+        Returns true if this item is the start of a new group,
+        where groups mean that some attribute has changed.  The getter
+        can be None (the item itself changes), an attribute name like
+        ``'.attr'``, a function, or a dict key or list index.
+        """
+        if self.first:
+            return True
+        return self._compare_group(self.item, self.previous, getter)
+
+    def last_group(self, getter=None):
+        """
+        Returns true if this item is the end of a new group,
+        where groups mean that some attribute has changed.  The getter
+        can be None (the item itself changes), an attribute name like
+        ``'.attr'``, a function, or a dict key or list index.
+        """
+        if self.last:
+            return True
+        return self._compare_group(self.item, self.__next__, getter)
+
+    def _compare_group(self, item, other, getter):
+        if getter is None:
+            return item != other
+        elif (isinstance(getter, basestring_)
+              and getter.startswith('.')):
+            getter = getter[1:]
+            if getter.endswith('()'):
+                getter = getter[:-2]
+                return getattr(item, getter)() != getattr(other, getter)()
+            else:
+                return getattr(item, getter) != getattr(other, getter)
+        elif hasattr(getter, '__call__'):
+            return getter(item) != getter(other)
+        else:
+            return item[getter] != other[getter]

Cython/Tempita/compat3.py

+import sys
+
+__all__ = ['b', 'basestring_', 'bytes', 'next', 'is_unicode']
+
+if sys.version < "3":
+    b = bytes = str
+    basestring_ = basestring
+else:
+
+    def b(s):
+        if isinstance(s, str):
+            return s.encode('latin1')
+        return bytes(s)
+    basestring_ = (bytes, str)
+    bytes = bytes
+text = str
+
+if sys.version < "3":
+
+    def next(obj):
+        return obj.next()
+else:
+    next = next
+
+if sys.version < "3":
+
+    def is_unicode(obj):
+        return isinstance(obj, unicode)
+else:
+
+    def is_unicode(obj):
+        return isinstance(obj, str)
+
+
+def coerce_text(v):
+    if not isinstance(v, basestring_):
+        if sys.version < "3":
+            attr = '__unicode__'
+        else:
+            attr = '__str__'
+        if hasattr(v, attr):
+            return unicode(v)
+        else:
+            return bytes(v)
+    return v

Cython/Utility/MemoryView_C.c

@@ -3,13 +3,13 @@
 /* memoryview slice struct */
 
 typedef struct {
-  struct %(memview_struct_name)s *memview;
+  struct {{memview_struct_name}} *memview;
   /* For convenience and faster access */
   char *data;
-  Py_ssize_t shape[%(max_dims)d];
-  Py_ssize_t strides[%(max_dims)d];
-  /* Py_ssize_t suboffsets[%(max_dims)d]; */
-} %(memviewslice_name)s;
+  Py_ssize_t shape[{{max_dims}}];
+  Py_ssize_t strides[{{max_dims}}];
+  Py_ssize_t suboffsets[{{max_dims}}];
+} {{memviewslice_name}};
 
 // UtilityProto: MemviewSliceInit
 
@@ -194,12 +194,11 @@ static int __Pyx_init_memviewslice(
     for(i=0; i<ndim; i++) {
         memviewslice->strides[i] = buf->strides[i];
         memviewslice->shape[i]   = buf->shape[i];
-        /*
         if(buf->suboffsets) {
             memviewslice->suboffsets[i] = buf->suboffsets[i];
         } else {
             memviewslice->suboffsets[i] = -1;
-        }*/
+        }
     }
 
     __Pyx_INCREF((PyObject *)memview);
@@ -221,7 +220,7 @@ no_fail:
 
 // UtilityCode: MemviewSliceCopyTemplate
 
-static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
+static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
 
     __Pyx_RefNannyDeclarations
     int i;
@@ -232,9 +231,9 @@ static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
     PyObject *temp_int = 0;
     struct __pyx_array_obj *array_obj = 0;
     struct __pyx_memoryview_obj *memview_obj = 0;
-    char *mode = "%(mode)s";
+    char *mode = "{{mode}}";
 
-    __Pyx_RefNannySetupContext("%(copy_name)s");
+    __Pyx_RefNannySetupContext("{{copy_name}}");
 
     shape_tuple = PyTuple_New((Py_ssize_t)(buf->ndim));
     if(unlikely(!shape_tuple)) {
@@ -252,13 +251,13 @@ static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
         }
     }
 
-    array_obj = __pyx_array_new(shape_tuple, %(sizeof_dtype)s, buf->format, mode);
+    array_obj = __pyx_array_new(shape_tuple, {{sizeof_dtype}}, buf->format, mode);
     if (unlikely(!array_obj)) {
         goto fail;
     }
     __Pyx_GOTREF(array_obj);
 
-    memview_obj = __pyx_memoryview_new((PyObject *) array_obj, %(contig_flag)s);
+    memview_obj = __pyx_memoryview_new((PyObject *) array_obj, {{contig_flag}});
     if (unlikely(!memview_obj)) {
         goto fail;
     }
@@ -270,7 +269,7 @@ static __Pyx_memviewslice %(copy_name)s(const __Pyx_memviewslice from_mvs) {
         goto fail;
     }
 
-    if (unlikely(-1 == %(copy_contents_name)s(&from_mvs, &new_mvs))) {
+    if (unlikely(-1 == {{copy_contents_name}}(&from_mvs, &new_mvs))) {
         /* PyErr_SetString(PyExc_RuntimeError,
             "Could not copy contents of memoryview slice."); */
         goto fail;

setup.py

@@ -274,6 +274,7 @@ packages = [
     'Cython.Build.Tests',
     'Cython.Compiler.Tests',
     'Cython.Utility',
+    'Cython.Tempita',
 ]
 
 if include_debugger:

tests/run/memslice_indexing.pyx

@@ -20,7 +20,10 @@ def create_array(shape, mode='c'):
 
 def slice_contig_indexing():
     """
-    >>> slice_contig_indexing()
+    >>> print "disabled"
+    disabled
+
+    slice_contig_indexing()
     98
     61
     98