py23 compat

Cython/Compiler/Code.pxd

@@ -2,7 +2,7 @@
 cimport cython
 
 cdef class UtilityCodeBase(object):
-    pass
+    cdef public object name
 
 cdef class UtilityCode(UtilityCodeBase):
     cdef public object proto

Cython/Compiler/Code.py

@@ -59,12 +59,8 @@ class UtilityCodeBase(object):
     # @classmethod
     def _add_utility(cls, utility, type, lines, begin_lineno):
         if utility:
-            if cls.is_cython_utility:
-                # Don't forget our line number
-                code = '' * begin_lineno + ''.join(lines)
-            else:
-                # line numbers are not important here
-                code = ''.join(lines)
+            # Remember line numbers as least until after templating
+            code = '' * begin_lineno + ''.join(lines)
 
             if type == 'Proto':
                 utility[0] = code
@@ -96,28 +92,34 @@ class UtilityCodeBase(object):
 
         f = Utils.open_source_file(filename, encoding='UTF-8')
         try:
-            for lineno, line in enumerate(f):
-                m = re.search(regex, line)
-                if m:
-                    cls._add_utility(utility, type, lines, begin_lineno)
-
-                    begin_lineno = lineno + 1
-                    name = m.group(1)
-                    if name.endswith(".proto"):
-                        name = name[:-6]
-                        type = 'Proto'
-                    else:
-                        type = 'Code'
-
-                    utility = utilities.setdefault(name, [None, None])
-                    utilities[name] = utility
-
-                    lines = []
-                else:
-                    lines.append(line)
+            all_lines = f.readlines() # py23
         finally:
             f.close()
 
+        for lineno, line in enumerate(all_lines):
+            # apparently 'line' may be without trailing newline
+            # (NormalisedNewlineStream.readlines())
+            line = line.rstrip() + '\n'
+
+            m = re.search(regex, line)
+            if m:
+                cls._add_utility(utility, type, lines, begin_lineno)
+
+                begin_lineno = lineno + 1
+                name = m.group(1)
+                if name.endswith(".proto"):
+                    name = name[:-6]
+                    type = 'Proto'
+                else:
+                    type = 'Code'
+
+                utility = utilities.setdefault(name, [None, None])
+                utilities[name] = utility
+
+                lines = []
+            else:
+                lines.append(line)
+
         if not utility:
             raise ValueError("Empty utility code file")
 
@@ -190,12 +192,19 @@ class UtilityCodeBase(object):
 
         proto, impl = utilities[util_code_name]
         if context is not None:
+            if '__name' not in context:
+                context['__name'] = util_code_name
+
             if proto:
                 proto = tempita.sub(proto, **context)
             if impl:
                 impl = tempita.sub(impl, **context)
 
-        return proto, impl
+        if cls.is_cython_utility:
+            # Remember line numbers
+            return proto, impl
+
+        return proto and proto.lstrip(), impl and impl.lstrip()
 
     load_as_string = classmethod(load_as_string)
 

Cython/Tempita/__init__.py

@@ -565,7 +565,7 @@ class TemplateDef(object):
             else:
                 raise TypeError(
                     'Extra position arguments: %s'
-                    % ', '.join(repr(v) for v in args))
+                    % ', '.join([repr(v) for v in args]))
         for name, value_expr in defaults.iteritems():
             if name not in values:
                 values[name] = self._template._eval(
@@ -660,6 +660,7 @@ def lex(s, name=None, trim_whitespace=True, line_offset=0, delimeters=None):
     chunks = []
     last = 0
     last_pos = (1, 1)
+
     token_re = re.compile(r'%s|%s' % (re.escape(delimeters[0]),
                                       re.escape(delimeters[1])))
     for match in token_re.finditer(s):

Cython/Utils.py

@@ -130,6 +130,8 @@ class NormalisedNewlineStream(object):
     while data:
         content.append(data)
         data = self.read(0x1000)
+
+    # TODO: FIXME: Shouldn't this return lines with their newline appended??
     return u''.join(content).split(u'\n')
 
 io = None

tests/run/memslice_indexing.pyx

@@ -20,7 +20,7 @@ def create_array(shape, mode='c'):
 
 def slice_contig_indexing():
     """
-    >>> print "disabled"
+    >>> print("disabled")
     disabled
 
     slice_contig_indexing()