merge

.hgignore

@@ -5,7 +5,9 @@ syntax: glob
 
 Cython/Compiler/Lexicon.pickle
 BUILD/
+build/
 .coverage
 *~
 *.orig
 *.rej
+*.dep

Cython/Compiler/Code.py

@@ -33,10 +33,11 @@ class CCodeWriter:
     def __init__(self, f):
         #self.f = open_new_file(outfile_name)
         self.f = f
+        self._write = f.write
         self.level = 0
         self.bol = 1
         self.marker = None
-        self.last_marker = 1
+        self.last_marker_line = 0
         self.label_counter = 1
         self.error_label = None
         self.filename_table = {}
@@ -49,14 +50,14 @@ class CCodeWriter:
             self.emit_marker()
         if code:
             self.put(code)
-        self.f.write("\n");
+        self._write("\n");
         self.bol = 1
     
     def emit_marker(self):
-        self.f.write("\n");
+        self._write("\n");
         self.indent()
-        self.f.write("/* %s */\n" % self.marker)
-        self.last_marker = self.marker
+        self._write("/* %s */\n" % self.marker[1])
+        self.last_marker_line = self.marker[0]
         self.marker = None
 
     def put(self, code):
@@ -65,7 +66,7 @@ class CCodeWriter:
             self.level += dl
         if self.bol:
             self.indent()
-        self.f.write(code)
+        self._write(code)
         self.bol = 0
         if dl > 0:
             self.level += dl
@@ -85,18 +86,19 @@ class CCodeWriter:
         self.putln("}")
     
     def indent(self):
-        self.f.write("  " * self.level)
+        self._write("  " * self.level)
 
     def get_py_version_hex(self, pyversion):
         return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]
 
-    def file_contents(self, source_desc):
+    def commented_file_contents(self, source_desc):
         try:
             return self.input_file_contents[source_desc]
         except KeyError:
-            F = [line.encode('ASCII', 'replace').replace(
-                    '*/', '*[inserted by cython to avoid comment closer]/')
-                 for line in source_desc.get_lines(decode=True)]
+            F = [u' * ' + line.rstrip().replace(
+                    u'*/', u'*[inserted by cython to avoid comment closer]/'
+                    ).encode('ASCII', 'replace') # + Py2 auto-decode to unicode
+                 for line in source_desc.get_lines()]
             self.input_file_contents[source_desc] = F
             return F
 
@@ -104,19 +106,18 @@ class CCodeWriter:
         if pos is None:
             return
         source_desc, line, col = pos
+        if self.last_marker_line == line:
+            return
         assert isinstance(source_desc, SourceDescriptor)
-        contents = self.file_contents(source_desc)
-
-        context = ''
-        for i in range(max(0,line-3), min(line+2, len(contents))):
-            s = contents[i]
-            if i+1 == line:   # line numbers in pyrex start counting up from 1
-                s = s.rstrip() + '             # <<<<<<<<<<<<<< ' + '\n'
-            context += " * " + s
-
-        marker = '"%s":%d\n%s' % (source_desc.get_description().encode('ASCII', 'replace'), line, context)
-        if self.last_marker != marker:
-            self.marker = marker
+        contents = self.commented_file_contents(source_desc)
+
+        lines = contents[max(0,line-3):line] # line numbers start at 1
+        lines[-1] += u'             # <<<<<<<<<<<<<<'
+        lines += contents[line:line+2]
+
+        marker = u'"%s":%d\n%s\n' % (
+            source_desc.get_escaped_description(), line, u'\n'.join(lines))
+        self.marker = (line, marker)
 
     def init_labels(self):
         self.label_counter = 0

Cython/Compiler/Errors.py

@@ -17,10 +17,15 @@ def context(position):
     source = position[0]
     assert not (isinstance(source, unicode) or isinstance(source, str)), (
         "Please replace filename strings with Scanning.FileSourceDescriptor instances %r" % source)
+    try:
         F = list(source.get_lines())
-    s = ''.join(F[min(0, position[1]-6):position[1]])
-    s += ' '*(position[2]-1) + '^'
-    s = '-'*60 + '\n...\n' + s + '\n' + '-'*60 + '\n'
+    except UnicodeDecodeError:
+        # file has an encoding problem
+        s = "[unprintable code]\n"
+    else:
+        s =''.join(F[max(0, position[1]-6):position[1]])
+        s = '...\n' + s + ' '*(position[2]-1) + '^\n'
+    s = '-'*60 + '\n' + s + '-'*60 + '\n'
     return s
     
 class CompileError(PyrexError):

Cython/Compiler/Nodes.py

@@ -153,24 +153,24 @@ class Node(object):
             self.body.annotate(code)
             
     def end_pos(self):
+        if not self.child_attrs:
+            return self.pos
         try:
             return self._end_pos
         except AttributeError:
-            flat = []
+            pos = self.pos
             for attr in self.child_attrs:
                 child = getattr(self, attr)
                 # Sometimes lists, sometimes nodes
                 if child is None:
                     pass
                 elif isinstance(child, list):
-                    flat += child
+                    for c in child:
+                        pos = max(pos, c.end_pos())
                 else:
-                    flat.append(child)
-            if len(flat) == 0:
-                self._end_pos = self.pos
-            else:
-                self._end_pos = max([child.end_pos() for child in flat])
-            return self._end_pos
+                    pos = max(pos, child.end_pos())
+            self._end_pos = pos
+            return pos
 
 
 class BlockNode:

Cython/Compiler/Scanning.py

@@ -209,9 +209,16 @@ class SourceDescriptor:
     """
     A SourceDescriptor should be considered immutable.
     """
+    _escaped_description = None
     def __str__(self):
         assert False # To catch all places where a descriptor is used directly as a filename
     
+    def get_escaped_description(self):
+        if self._escaped_description is None:
+            self._escaped_description = \
+                self.get_description().encode('ASCII', 'replace').decode("ASCII")
+        return self._escaped_description
+
 class FileSourceDescriptor(SourceDescriptor):
     """
     Represents a code source. A code source is a more generic abstraction
@@ -223,16 +230,8 @@ class FileSourceDescriptor(SourceDescriptor):
     def __init__(self, filename):
         self.filename = filename
     
-    def get_lines(self, decode=False):
-        # decode is True when called from Code.py (which reserializes in a standard way to ASCII),
-        # while decode is False when called from Errors.py.
-        #
-        # Note that if changing Errors.py in this respect, raising errors over wrong encoding
-        # will no longer be able to produce the line where the encoding problem occurs ...
-        if decode:
+    def get_lines(self):
         return Utils.open_source_file(self.filename)
-        else:
-            return open(self.filename)
     
     def get_description(self):
         return self.filename
@@ -258,7 +257,7 @@ class StringSourceDescriptor(SourceDescriptor):
         self.name = name
         self.codelines = [x + "\n" for x in code.split("\n")]
     
-    def get_lines(self, decode=False):
+    def get_lines(self):
         return self.codelines
     
     def get_description(self):

Makefile

-VERSION = 0.9.6.3
 PYTHON?=python
 
-version:
-	@echo "Setting version to $(VERSION)"
-	@echo "version = '$(VERSION)'" > Cython/Compiler/Version.py
-
 clean:
 	@echo Cleaning Source
 	@rm -f *.pyc */*.pyc */*/*.pyc 

setup.py

-from distutils.core import setup
+from distutils.core import setup, Extension
 from distutils.sysconfig import get_python_lib
-import os, sys
+import os, os.path
 import sys
 from Cython.Compiler.Version import version
 
@@ -22,6 +22,31 @@ if os.name == "posix":
 else:
     scripts = ["cython.py"]
 
+try:
+    sys.argv.remove("--no-compile")
+except ValueError:
+    try:
+        from Cython.Compiler.Main import compile
+        source_root = os.path.dirname(__file__)
+        compiled_modules = ["Cython.Plex.Scanners"]
+        extensions = []
+        for module in compiled_modules:
+            source_file = os.path.join(source_root, *module.split('.'))
+            print("Compiling module %s ..." % module)
+            result = compile(source_file + ".py")
+            if result.c_file:
+                extensions.append(
+                    Extension(module, sources = [result.c_file])
+                    )
+            else:
+                print("Compilation failed")
+        if extensions:
+            setup_args['ext_modules'] = extensions
+    except Exception:
+        print("ERROR: %s" % sys.exc_info()[1])
+        print("Extension module compilation failed, using plain Python implementation")
+
+
 setup(
   name = 'Cython', 
   version = version,

tests/run/r_uintindex.pyx

+__doc__ = u"""
+>>> print idx_uint( ["buckle", "my", "shoe"], 2)
+shoe
+>>> print idx_ulong(["buckle", "my", "shoe"], 2)
+shoe
+"""
+
+def idx_ulong(seq, i):
+    cdef unsigned long u
+    u = i
+    return seq[u]
+
+def idx_uint(seq, i):
+    cdef unsigned int u
+    u = i
+    return seq[u]