Merge package and compiler repositories.

544f9242 · Robert Bradshaw · bb088c2a · f2fdd6bc · 544f9242 · 544f9242
Commit 544f9242 authored Apr 03, 2008 by Robert Bradshaw
45 changed files
--- a/Cython/Compiler/Annotate.py
+++ b/Cython/Compiler/Annotate.py
+# Note: Work in progress
+
+import os
+import re
+import time
+from StringIO import StringIO
+
+import Version
+from Code import CCodeWriter
+
+# need one-characters subsitutions (for now) so offsets aren't off
+special_chars = [('<', '\xF0', '&lt;'),
+                 ('>', '\xF1', '&gt;'), 
+                 ('&', '\xF2', '&amp;')]
+
+class AnnotationCCodeWriter(CCodeWriter):
+
+    def __init__(self, f):
+        CCodeWriter.__init__(self, self)
+        self.buffer = StringIO()
+        self.real_f = f
+        self.annotations = []
+        self.last_pos = None
+        self.code = {}
+        
+    def getvalue(self):
+        return self.real_f.getvalue()
+        
+    def write(self, s):
+        self.real_f.write(s)
+        self.buffer.write(s)
+        
+    def mark_pos(self, pos):
+#        if pos is not None:
+#            CCodeWriter.mark_pos(self, pos)
+#        return
+        if self.last_pos:
+            try:
+                code = self.code[self.last_pos[1]]
+            except KeyError:
+                code = ""
+            self.code[self.last_pos[1]] = code + self.buffer.getvalue()
+        self.buffer = StringIO()
+        self.last_pos = pos
+
+    def annotate(self, pos, item):
+        self.annotations.append((pos, item))
+        
+    def save_annotation(self, filename):
+        self.mark_pos(None)
+        f = open(filename)
+        lines = f.readlines()
+        for k in range(len(lines)):
+            line = lines[k]
+            for c, cc, html in special_chars:
+                line = line.replace(c, cc)
+            lines[k] = line
+        f.close()
+        all = []
+        for pos, item in self.annotations:
+            if pos[0] == filename:
+                start = item.start()
+                size, end = item.end()
+                if size:
+                    all.append((pos, start))
+                    all.append(((filename, pos[1], pos[2]+size), end))
+                else:
+                    all.append((pos, start+end))
+                
+        all.sort()
+        all.reverse()
+        for pos, item in all:
+            _, line_no, col = pos
+            line_no -= 1
+            col += 1
+            line = lines[line_no]
+            lines[line_no] = line[:col] + item + line[col:]
+        
+        f = open("%s.html" % filename, "w")
+        f.write('<html>\n')
+        f.write("""
+<head>
+<style type="text/css">
+
+body { font-family: courier; font-size: 12; }
+
+.code  { font-size: 9; color: #444444; display: none; margin-left: 20px; }
+.py_api  { color: red; }
+.pyx_api  { color: #FF3000; }
+.py_macro_api  { color: #FF8000; }
+.error_goto  { color: #FF8000; }
+
+.tag  {  }
+
+.coerce  { color: #008000; border: 1px dotted #008000 }
+
+.py_attr { color: #FF0000; font-weight: bold; }
+.c_attr  { color: #0000FF; }
+
+.py_call { color: #FF0000; font-weight: bold; }
+.c_call  { color: #0000FF; }
+
+.line { margin: 0em }
+
+</style>
+<script>
+function toggleDiv(id) {
+    theDiv = document.getElementById(id);
+    if (theDiv.style.display == 'none') theDiv.style.display = 'block';
+    else theDiv.style.display = 'none';
+}
+</script>
+</head>
+        """)
+        f.write('<body>\n')
+        f.write('<p>Generated by Cython %s on %s\n' % (Version.version, time.asctime()))
+        c_file = os.path.basename(filename)[:-3] + 'c'
+        f.write('<p>Raw output: <a href="%s">%s</a>\n' % (c_file, c_file))
+        k = 0
+        
+        py_c_api = re.compile('(Py[A-Z][a-z]+_[A-Z][a-z][A-Za-z_]+)')
+        pyx_api = re.compile('(__Pyx[A-Za-z_]+)\(')
+        py_marco_api = re.compile('(Py[A-Za-z]*_[A-Z][A-Z_]+)')
+        error_goto = re.compile(r'((; *if .*)? \{__pyx_filename = .*goto __pyx_L\w+;\})')
+        
+        for line in lines:
+
+            k += 1
+            try:
+                code = self.code[k]
+            except KeyError:
+                code = ''
+                
+            code, c_api_calls = py_c_api.subn(r"<span class='py_api'>\1</span>", code)
+            code, pyx_api_calls = pyx_api.subn(r"<span class='pyx_api'>\1</span>(", code)
+            code, macro_api_calls = py_marco_api.subn(r"<span class='py_macro_api'>\1</span>", code)
+            code, error_goto_calls = error_goto.subn(r"<span class='error_goto'>\1</span>", code)
+            
+            code = code.replace("<span class='error_goto'>;", ";<span class='error_goto'>")
+            
+            color = "FFFF%02x" % int(255/(1+(5*c_api_calls+2*pyx_api_calls+macro_api_calls)/10.0))
+            f.write("<pre class='line' style='background-color: #%s' onclick='toggleDiv(\"line%s\")'>" % (color, k))
+
+            f.write(" %d: " % k)
+            for c, cc, html in special_chars:
+                line = line.replace(cc, html)
+            f.write(line.rstrip())
+                
+            f.write('</pre>\n')
+            f.write("<pre id='line%s' class='code' style='background-color: #%s'>%s</pre>" % (k, color, code))
+        f.write('</body></html>\n')
+        f.close()
+        
+
+# TODO: make this cleaner
+def escape(raw_string):
+    raw_string = raw_string.replace("\'", r"&#146;")
+    raw_string = raw_string.replace('\"', r'&quot;')
+    raw_string = raw_string.replace('\n', r'<br>\n')
+    raw_string = raw_string.replace('\t', r'\t')
+    return raw_string
+
+
+class AnnotationItem:
+    
+    def __init__(self, style, text, tag="", size=0):
+        self.style = style
+        self.text = text
+        self.tag = tag
+        self.size = size
+        
+    def start(self):
+        return "<span class='tag %s' title='%s'>%s" % (self.style, self.text, self.tag)
+    
+    def end(self):
+        return self.size, "</span>"
--- a/Cython/Compiler/Builtin.py
+++ b/Cython/Compiler/Builtin.py
+#
+#   Pyrex - Builtin Definitions
+#
+
+from Symtab import BuiltinScope
+from TypeSlots import Signature
+
+builtin_function_table = [
+    # name,        args,   return,  C API func,           py equiv = "*"
+    ('abs',        "O",    "O",     "PyNumber_Absolute"),
+    #('chr',       "",     "",      ""),
+    #('cmp', "",   "",     "",      ""), # int PyObject_Cmp(PyObject *o1, PyObject *o2, int *result)
+    #('compile',   "",     "",      ""), # PyObject* Py_CompileString(	char *str, char *filename, int start)
+    ('delattr',    "OO",   "r",     "PyObject_DelAttr"),
+    ('dir',        "O",    "O",     "PyObject_Dir"),
+    ('divmod',     "OO",   "O",     "PyNumber_Divmod"),
+    #('eval',      "",     "",      ""),
+    #('execfile',  "",     "",      ""),
+    #('filter',    "",     "",      ""),
+    ('getattr',    "OO",   "O",     "PyObject_GetAttr"),
+    ('getattr3',   "OOO",  "O",     "__Pyx_GetAttr3",       "getattr"),
+    ('hasattr',    "OO",   "b",     "PyObject_HasAttr"),
+    ('hash',       "O",    "l",     "PyObject_Hash"),
+    #('hex',       "",     "",      ""),
+    #('id',        "",     "",      ""),
+    #('input',     "",     "",      ""),
+    ('intern',     "s",    "O",     "PyString_InternFromString"),
+    ('isinstance', "OO",   "b",     "PyObject_IsInstance"),
+    ('issubclass', "OO",   "b",     "PyObject_IsSubclass"),
+    ('iter',       "O",    "O",     "PyObject_GetIter"),
+    ('len',        "O",    "Z",     "PyObject_Length"),
+    #('map',       "",     "",      ""),
+    #('max',       "",     "",      ""),
+    #('min',       "",     "",      ""),
+    #('oct',       "",     "",      ""),
+    # Not worth doing open, when second argument would become mandatory
+    #('open',       "ss",   "O",     "PyFile_FromString"),
+    #('ord',       "",     "",      ""),
+    ('pow',        "OOO",  "O",     "PyNumber_Power"),
+    #('range',     "",     "",      ""),
+    #('raw_input', "",     "",      ""),
+    #('reduce',    "",     "",      ""),
+    ('reload',     "O",    "O",     "PyImport_ReloadModule"),
+    ('repr',       "O",    "O",     "PyObject_Repr"),
+    #('round',     "",     "",      ""),
+    ('setattr',    "OOO",  "r",     "PyObject_SetAttr"),
+    #('sum',       "",     "",      ""),
+    #('unichr',    "",     "",      ""),
+    #('unicode',   "",     "",      ""),
+    #('vars',      "",     "",      ""),
+    #('zip',       "",     "",      ""),
+    #  Can't do these easily until we have builtin type entries.
+    #('typecheck',  "OO",   "i",     "PyObject_TypeCheck", False),
+    #('issubtype',  "OO",   "i",     "PyType_IsSubtype",   False),
+]
+
+# Builtin types
+#  bool
+#  buffer
+#  classmethod
+#  dict
+#  enumerate
+#  file
+#  float
+#  int
+#  list
+#  long
+#  object
+#  property
+#  slice
+#  staticmethod
+#  super
+#  str
+#  tuple
+#  type
+#  xrange
+
+getattr3_utility_code = ["""
+static PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); /*proto*/
+""","""
+static PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) {
+    PyObject *r = PyObject_GetAttr(o, n);
+    if (!r) {
+        if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+            goto bad;
+        PyErr_Clear();
+        r = d;
+        Py_INCREF(d);
+    }
+    return r;
+bad:
+    return 0;
+}
+"""]
+
+builtin_utility_code = {
+    'getattr3': getattr3_utility_code,
+}
+
+builtin_scope = BuiltinScope()
+
+def declare_builtin_func(name, args, ret, cname, py_equiv = "*"):
+    sig = Signature(args, ret)
+    type = sig.function_type()
+    utility = builtin_utility_code.get(name)
+    builtin_scope.declare_builtin_cfunction(name, type, cname, py_equiv, utility)
+
+def init_builtin_funcs():
+    for desc in builtin_function_table:
+        declare_builtin_func(*desc)
+
+def init_builtins():
+    init_builtin_funcs()
+
+init_builtins()
--- a/Cython/Compiler/CmdLine.py
+++ b/Cython/Compiler/CmdLine.py
+#
+#   Cython - Command Line Parsing
+#
+
+import sys
+import Options
+import Transform
+
+usage = """\
+Cython (http://cython.org) is a compiler for code written in the
+Cython language.  Cython is based on Pyrex by Greg Ewing.
+
+Usage: cython [options] sourcefile.pyx ...
+
+Options:
+  -v, --version                  Display version number of cython compiler
+  -l, --create-listing           Write error messages to a listing file
+  -I, --include-dir <directory>  Search for include files in named directory
+                                 (multiply include directories are allowed).
+  -o, --output-file <filename>   Specify name of generated C file
+  -p, --embed-positions          If specified, the positions in Cython files of each
+                                 function definition is embedded in its docstring.
+  -z, --pre-import <module>      If specified, assume undeclared names in this 
+                                 module. Emulates the behavior of putting 
+                                 "from <module> import *" at the top of the file. 
+  --incref-local-binop           Force local an extra incref on local variables before
+                                 performing any binary operations.
+  --cleanup <level>              Release interned objects on python exit, for memory debugging. 
+                                 Level indicates aggressiveness, default 0 releases nothing. 
+  -D, --no-docstrings            Remove docstrings.
+  -a, --annotate                 Produce an colorized version of the source.
+  --convert-range                Convert for loops using range() function to for...from loops. 
+  --cplus                        Output a c++ rather than c file.
+"""
+#The following experimental options are supported only on MacOSX:
+#  -C, --compile    Compile generated .c file to .o file
+#  -X, --link       Link .o file to produce extension module (implies -C)
+#  -+, --cplus      Use C++ compiler for compiling and linking
+#  Additional .o files to link may be supplied when using -X."""
+
+#The following options are very experimental and is used for plugging in code
+#into different transform stages.
+#  -T phase:factory At the phase given, hand off the tree to the transform returned
+#                   when calling factory without arguments. Factory should be fully
+#                   specified (ie Module.SubModule.factory) and the containing module
+#                   will be imported. This option can be repeated to add more transforms,
+#                   transforms for the same phase will be used in the order they are given.
+
+def bad_usage():
+    print >>sys.stderr, usage
+    sys.exit(1)
+
+def parse_command_line(args):
+
+    def parse_add_transform(transforms, param):
+        def import_symbol(fqn):
+            modsplitpt = fqn.rfind(".")
+            if modsplitpt == -1: bad_usage()
+            modulename = fqn[:modsplitpt]
+            symbolname = fqn[modsplitpt+1:]
+            module = __import__(modulename, globals(), locals(), [symbolname])
+            return getattr(module, symbolname)
+    
+        stagename, factoryname = param.split(":")
+        if not stagename in Transform.PHASES:
+            bad_usage()
+        factory = import_symbol(factoryname)
+        transform = factory()
+        transforms[stagename].append(transform)
+    
+    from Cython.Compiler.Main import \
+        CompilationOptions, default_options
+
+    def pop_arg():
+        if args:
+            return args.pop(0)
+        else:
+            bad_usage()
+    
+    def get_param(option):
+        tail = option[2:]
+        if tail:
+            return tail
+        else:
+            return pop_arg()
+
+    options = CompilationOptions(default_options)
+    sources = []
+    while args:
+        if args[0].startswith("-"):
+            option = pop_arg()
+            if option in ("-v", "--version"):
+                options.show_version = 1
+            elif option in ("-l", "--create-listing"):
+                options.use_listing_file = 1
+            elif option in ("-C", "--compile"):
+                options.c_only = 0
+            elif option in ("-X", "--link"):
+                options.c_only = 0
+                options.obj_only = 0
+            elif option in ("-+", "--cplus"):
+                options.cplus = 1
+            elif option.startswith("-I"):
+                options.include_path.append(get_param(option))
+            elif option == "--include-dir":
+                options.include_path.append(pop_arg())
+            elif option in ("-o", "--output-file"):
+                options.output_file = pop_arg()
+            elif option in ("-p", "--embed-positions"):
+                Options.embed_pos_in_docstring = 1
+            elif option in ("-z", "--pre-import"):
+                Options.pre_import = pop_arg()
+            elif option == "--incref-local-binop":
+                Options.incref_local_binop = 1
+            elif option == "--cleanup":
+                Options.generate_cleanup_code = int(pop_arg())
+            elif option in ("-D", "--no-docstrings"):
+                Options.docstrings = False
+            elif option in ("-a", "--annotate"):
+                Options.annotate = True
+            elif option == "--convert-range":
+                Options.convert_range = True
+            elif option.startswith("-T"):
+                parse_add_transform(options.transforms, get_param(option))
+                # Note: this can occur multiple times, each time appends
+            else:
+                bad_usage()
+        else:
+            arg = pop_arg()
+            if arg.endswith(".pyx"):
+                sources.append(arg)
+            elif arg.endswith(".o"):
+                options.objects.append(arg)
+            else:
+                print >>sys.stderr, \
+                    "cython: %s: Unknown filename suffix" % arg
+    if options.objects and len(sources) > 1:
+        print >>sys.stderr, \
+            "cython: Only one source file allowed together with .o files"
+    if options.use_listing_file and len(sources) > 1:
+        print >>sys.stderr, \
+            "cython: Only one source file allowed when using -o"
+        sys.exit(1)
+    if len(sources) == 0 and not options.show_version:
+        bad_usage()
+    return options, sources
+
--- a/Cython/Compiler/Code.py
+++ b/Cython/Compiler/Code.py
--- a/Cython/Compiler/ControlFlow.py
+++ b/Cython/Compiler/ControlFlow.py
+import bisect
+
+# This module keeps track of arbitrary "states" at any point of the code. 
+# A state is considered known if every path to the given point agrees on
+# its state, otherwise it is None (i.e. unknown). 
+
+# It might be useful to be able to "freeze" the set of states by pushing 
+# all state changes to the tips of the trees for fast reading. Perhaps this
+# could be done on get_state, clearing the cache on set_state (assuming 
+# incoming is immutable). 
+
+# This module still needs a lot of work, and probably should totally be 
+# redesigned. It doesn't take return, raise, continue, or break into 
+# account. 
+
+class ControlFlow:
+
+    def __init__(self, start_pos, incoming, parent):
+        self.start_pos = start_pos
+        self.incoming = incoming
+        if parent is None and incoming is not None:
+            parent = incoming.parent
+        self.parent = parent
+        self.tip = {}
+        self.end_pos = ((),)
+        
+    def start_branch(self, pos):
+        self.end_pos = pos
+        branch_point = BranchingControlFlow(pos, self)
+        if self.parent is not None:
+            self.parent.branches[-1] = branch_point
+        return branch_point.branches[0]
+    
+    def next_branch(self, pos):
+        self.end_pos = pos
+        return self.parent.new_branch(pos)
+        
+    def finish_branch(self, pos):
+        self.end_pos = pos
+        self.parent.end_pos = pos
+        return LinearControlFlow(pos, self.parent)
+        
+    def get_state(self, item, pos=((),())):
+        return self.get_pos_state(item, pos)[1]
+        
+    def get_pos_state(self, item, pos=((),())):
+        # do some caching
+        if pos > self.end_pos:
+            try:
+                return self.tip[item]
+            except KeyError:
+                self.tip[item] = pos_state = self._get_pos_state(item, pos)
+                return pos_state
+        else:
+            return self._get_pos_state(item, pos)
+        
+class LinearControlFlow(ControlFlow):
+
+    def __init__(self, start_pos=(), incoming=None, parent=None):
+        ControlFlow.__init__(self, start_pos, incoming, parent)
+        self.events = {}
+            
+    def set_state(self, pos, item, state):
+        if self.tip.has_key(item):
+            del self.tip[item]
+        if pos < self.start_pos:
+            if self.incoming is not None:
+                self.incoming.set_state(pos, item, state)
+        else:
+            if self.events.has_key(item):
+                event_list = self.events[item]
+            else:
+                event_list = []
+                self.events[item] = event_list
+            bisect.insort(event_list, (pos, state))
+            
+        
+    def _get_pos_state(self, item, pos):
+        if pos > self.start_pos:
+            if self.events.has_key(item):
+                event_list = self.events[item]
+                for event in event_list[::-1]:
+                    if event[0] < pos:
+                        return event
+                        
+        if self.incoming is not None:
+            return self.incoming.get_pos_state(item, pos)
+        
+        else:
+            return None, None
+            
+    def to_string(self, indent='', limit=None):
+    
+        if len(self.events) == 0:
+            s = indent + "[no state changes]"
+            
+        else:
+            all = []
+            for item, event_list in self.events.items():
+                for pos, state in event_list:
+                    all.append((indent, pos, item, state))
+            all.sort()
+            all = ["%s%s: %s <- %s" % data for data in all]
+            s = "\n".join(all)
+        if self.incoming is not limit and self.incoming is not None:
+            s = "%s\n%s" % (self.incoming.to_string(indent, limit=limit), s)
+        return s
+    
+    
+class BranchingControlFlow(ControlFlow):
+    
+    def __init__(self, start_pos, incoming, parent=None):
+        ControlFlow.__init__(self, start_pos, incoming, parent)
+        self.branches = [LinearControlFlow(start_pos, incoming, parent=self)]
+        self.branch_starts = [start_pos]
+        
+    def set_state(self, pos, item, state):
+    
+        if self.tip.has_key(item):
+            del self.tip[item]
+        
+        if pos < self.start_pos:
+            self.incoming.set_state(pos, item, state)
+        else:
+            for branch_pos, branch in zip(self.branch_starts[::-1], self.branches[::-1]):
+                if pos >= branch_pos:
+                    branch.set_state(pos, item, state)
+                    return
+    
+    def _get_pos_state(self, item, pos):
+        if pos <= self.start_pos:
+            return self.incoming.get_pos_state(item, pos)
+        elif pos < self.end_pos:
+            for branch_pos, branch in zip(self.branch_starts[::-1], self.branches[::-1]):
+                if pos >= branch_pos:
+                    return branch.get_pos_state(item, pos)
+        else:
+            last_pos, last_state = self.branches[0].get_pos_state(item, pos)
+            if last_state is None:
+                return None, None
+            for branch in self.branches[1:]:
+                other_pos, other_state = branch.get_pos_state(item, pos)
+                if other_state is None or other_state != last_state:
+                    return None, None
+                elif last_pos is not other_pos:
+                    last_pos = max(last_pos, other_pos)
+            return last_pos, last_state
+
+    def new_branch(self, pos):
+        self.branches.append(LinearControlFlow(pos, self.incoming, parent=self))
+        self.branch_starts.append(pos)
+        return self.branches[-1]
+
+    def to_string(self, indent='', limit=None):
+        join = "\n%sor\n" % indent
+        s = join.join([branch.to_string(indent+"    ", limit=self.incoming) for branch in self.branches])
+        if self.incoming is not limit and self.incoming is not None:
+            s = "%s\n%s" % (self.incoming.to_string(indent, limit=limit), s)
+        return s
+        
+    
\ No newline at end of file
--- a/Cython/Compiler/DebugFlags.py
+++ b/Cython/Compiler/DebugFlags.py
+debug_disposal_code = 0
+debug_temp_alloc = 0
+debug_coercion = 0
+
--- a/Cython/Compiler/Errors.py
+++ b/Cython/Compiler/Errors.py
+#
+#   Pyrex - Errors
+#
+
+import sys
+from Cython.Utils import open_new_file
+
+
+class PyrexError(Exception):
+    pass
+
+class PyrexWarning(Exception):
+    pass
+
+def context(position):
+    F = open(position[0]).readlines()
+    s = ''.join(F[position[1]-6:position[1]])
+    s += ' '*(position[2]-1) + '^'
+    s = '-'*60 + '\n...\n' + s + '\n' + '-'*60 + '\n'
+    return s
+
+class CompileError(PyrexError):
+    
+    def __init__(self, position = None, message = ""):
+        self.position = position
+        self.message = message
+        if position:
+            pos_str = "%s:%d:%d: " % position
+            cont = context(position)
+        else:
+            pos_str = ""
+            cont = ''
+        Exception.__init__(self, '\nError converting Pyrex file to C:\n' + cont + '\n' + pos_str + message )
+
+class CompileWarning(PyrexWarning):
+    
+    def __init__(self, position = None, message = ""):
+        self.position = position
+        self.message = message
+        if position:
+            pos_str = "%s:%d:%d: " % position
+        else:
+            pos_str = ""
+        Exception.__init__(self, pos_str + message)
+
+
+class InternalError(Exception):
+    # If this is ever raised, there is a bug in the compiler.
+    
+    def __init__(self, message):
+        Exception.__init__(self, "Internal compiler error: %s"
+            % message)
+            
+
+listing_file = None
+num_errors = 0
+echo_file = None
+
+def open_listing_file(path, echo_to_stderr = 1):
+    # Begin a new error listing. If path is None, no file
+    # is opened, the error counter is just reset.
+    global listing_file, num_errors, echo_file
+    if path is not None:
+        listing_file = open_new_file(path)
+    else:
+        listing_file = None
+    if echo_to_stderr:
+        echo_file = sys.stderr
+    else:
+        echo_file = None
+    num_errors = 0
+
+def close_listing_file():
+    global listing_file
+    if listing_file:
+        listing_file.close()
+        listing_file = None
+
+def error(position, message):
+    #print "Errors.error:", repr(position), repr(message) ###
+    global num_errors
+    err = CompileError(position, message)
+    line = "%s\n" % err
+    if listing_file:
+        listing_file.write(line)
+    if echo_file:
+        echo_file.write(line)
+    num_errors = num_errors + 1
+    return err
+
+LEVEL=1 # warn about all errors level 1 or higher
+
+def warning(position, message, level=0):
+    if level < LEVEL:
+        return
+    warn = CompileWarning(position, message)
+    line = "warning: %s\n" % warn
+    if listing_file:
+        listing_file.write(line)
+    if echo_file:
+        echo_file.write(line)
+    return warn
--- a/Cython/Compiler/ExprNodes.py
+++ b/Cython/Compiler/ExprNodes.py
--- a/Cython/Compiler/Lexicon.py
+++ b/Cython/Compiler/Lexicon.py
+#
+#   Pyrex Scanner - Lexical Definitions
+#
+#   Changing anything in this file will cause Lexicon.pickle
+#   to be rebuilt next time pyrexc is run.
+#
+
+string_prefixes = "cCrRuU"
+
+def make_lexicon():
+    from Cython.Plex import \
+        Str, Any, AnyBut, AnyChar, Rep, Rep1, Opt, Bol, Eol, Eof, \
+        TEXT, IGNORE, State, Lexicon
+    from Scanning import Method
+
+    letter = Any("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_")
+    digit = Any("0123456789")
+    octdigit = Any("01234567")
+    hexdigit = Any("0123456789ABCDEFabcdef")
+    indentation = Bol + Rep(Any(" \t"))
+    
+    decimal = Rep1(digit)
+    dot = Str(".")
+    exponent = Any("Ee") + Opt(Any("+-")) + decimal
+    decimal_fract = (decimal + dot + Opt(decimal)) | (dot + decimal)
+    
+    name = letter + Rep(letter | digit)
+    intconst = decimal | (Str("0x") + Rep1(hexdigit))
+    longconst = intconst + Str("L")
+    fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent)
+    imagconst = (intconst | fltconst) + Any("jJ")
+    
+    sq_string = (
+        Str("'") + 
+        Rep(AnyBut("\\\n'") | (Str("\\") + AnyChar)) + 
+        Str("'")
+    )
+    
+    dq_string = (
+        Str('"') + 
+        Rep(AnyBut('\\\n"') | (Str("\\") + AnyChar)) + 
+        Str('"')
+    )
+    
+    non_sq = AnyBut("'") | (Str('\\') + AnyChar)
+    tsq_string = (
+        Str("'''")
+        + Rep(non_sq | (Str("'") + non_sq) | (Str("''") + non_sq)) 
+        + Str("'''")
+    )
+    
+    non_dq = AnyBut('"') | (Str('\\') + AnyChar)
+    tdq_string = (
+        Str('"""')
+        + Rep(non_dq | (Str('"') + non_dq) | (Str('""') + non_dq)) 
+        + Str('"""')
+    )
+    stringlit = Opt(Any(string_prefixes)) + (sq_string | dq_string | tsq_string| tdq_string)
+    
+    beginstring = Opt(Any(string_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
+    two_oct = octdigit + octdigit
+    three_oct = octdigit + octdigit + octdigit
+    two_hex = hexdigit + hexdigit
+    four_hex = two_hex + two_hex
+    escapeseq = Str("\\") + (two_oct | three_oct | two_hex |
+                             Str('u') + four_hex | Str('x') + two_hex | AnyChar)
+    
+    bra = Any("([{")
+    ket = Any(")]}")
+    punct = Any(":,;+-*/|&<>=.%`~^?")
+    diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=", "//")
+    spaces = Rep1(Any(" \t\f"))
+    comment = Str("#") + Rep(AnyBut("\n"))
+    escaped_newline = Str("\\\n")
+    lineterm = Eol + Opt(Str("\n"))
+    
+    return Lexicon([
+        (name, 'IDENT'),
+        (intconst, 'INT'),
+        (longconst, 'LONG'),
+        (fltconst, 'FLOAT'),
+        (imagconst, 'IMAG'),
+        (punct | diphthong, TEXT),
+        
+        (bra, Method('open_bracket_action')),
+        (ket, Method('close_bracket_action')),
+        (lineterm, Method('newline_action')),
+    
+        #(stringlit, 'STRING'),
+        (beginstring, Method('begin_string_action')),
+        
+        (comment, IGNORE),
+        (spaces, IGNORE),
+        (escaped_newline, IGNORE),
+        
+        State('INDENT', [
+            (Opt(spaces) + Opt(comment) + lineterm, IGNORE),
+            (indentation, Method('indentation_action')),
+            (Eof, Method('eof_action'))
+        ]),
+        
+        State('SQ_STRING', [
+            (escapeseq, 'ESCAPE'),
+            (Rep1(AnyBut("'\"\n\\")), 'CHARS'),
+            (Str('"'), 'CHARS'),
+            (Str("\n"), Method('unclosed_string_action')),
+            (Str("'"), Method('end_string_action')),
+            (Eof, 'EOF')
+        ]),
+        
+        State('DQ_STRING', [
+            (escapeseq, 'ESCAPE'),
+            (Rep1(AnyBut('"\n\\')), 'CHARS'),
+            (Str("'"), 'CHARS'),
+            (Str("\n"), Method('unclosed_string_action')),
+            (Str('"'), Method('end_string_action')),
+            (Eof, 'EOF')
+        ]),
+        
+        State('TSQ_STRING', [
+            (escapeseq, 'ESCAPE'),
+            (Rep1(AnyBut("'\"\n\\")), 'CHARS'),
+            (Any("'\""), 'CHARS'),
+            (Str("\n"), 'NEWLINE'),
+            (Str("'''"), Method('end_string_action')),
+            (Eof, 'EOF')
+        ]),
+        
+        State('TDQ_STRING', [
+            (escapeseq, 'ESCAPE'),
+            (Rep1(AnyBut('"\'\n\\')), 'CHARS'),
+            (Any("'\""), 'CHARS'),
+            (Str("\n"), 'NEWLINE'),
+            (Str('"""'), Method('end_string_action')),
+            (Eof, 'EOF')
+        ]),
+        
+        (Eof, Method('eof_action'))
+        ],
+        
+        # FIXME: Plex 1.9 needs different args here from Plex 1.1.4
+        #debug_flags = scanner_debug_flags,
+        #debug_file = scanner_dump_file
+        )
+
--- a/Cython/Compiler/Main.py
+++ b/Cython/Compiler/Main.py
--- a/Cython/Compiler/ModuleNode.py
+++ b/Cython/Compiler/ModuleNode.py
--- a/Cython/Compiler/Naming.py
+++ b/Cython/Compiler/Naming.py
+#
+#   Pyrex - C naming conventions
+#
+#
+#   Prefixes for generating C names.
+#   Collected here to facilitate ensuring uniqueness.
+#
+
+pyrex_prefix    = "__pyx_"
+
+builtin_prefix    = pyrex_prefix + "builtin_"
+arg_prefix        = pyrex_prefix + "arg_"
+funcdoc_prefix    = pyrex_prefix + "doc_"
+enum_prefix       = pyrex_prefix + "e_"
+func_prefix       = pyrex_prefix + "f_"
+pyfunc_prefix     = pyrex_prefix + "pf_"
+gstab_prefix      = pyrex_prefix + "getsets_"
+prop_get_prefix   = pyrex_prefix + "getprop_"
+const_prefix      = pyrex_prefix + "k"
+label_prefix      = pyrex_prefix + "L"
+pymethdef_prefix  = pyrex_prefix + "mdef_"
+methtab_prefix    = pyrex_prefix + "methods_"
+memtab_prefix     = pyrex_prefix + "members_"
+interned_prefix   = pyrex_prefix + "n_"
+interned_num_prefix = pyrex_prefix + "int_"
+objstruct_prefix  = pyrex_prefix + "obj_"
+typeptr_prefix    = pyrex_prefix + "ptype_"
+prop_set_prefix   = pyrex_prefix + "setprop_"
+type_prefix       = pyrex_prefix + "t_"
+typeobj_prefix    = pyrex_prefix + "type_"
+var_prefix        = pyrex_prefix + "v_"
+vtable_prefix     = pyrex_prefix + "vtable_"
+vtabptr_prefix    = pyrex_prefix + "vtabptr_"
+vtabstruct_prefix = pyrex_prefix + "vtabstruct_"
+opt_arg_prefix    = pyrex_prefix + "opt_args_"
+
+args_cname       = pyrex_prefix + "args"
+kwdlist_cname    = pyrex_prefix + "argnames"
+obj_base_cname   = pyrex_prefix + "base"
+builtins_cname   = pyrex_prefix + "b"
+preimport_cname  = pyrex_prefix + "i"
+moddict_cname    = pyrex_prefix + "d"
+dummy_cname      = pyrex_prefix + "dummy"
+filename_cname   = pyrex_prefix + "filename"
+filetable_cname  = pyrex_prefix + "f"
+filenames_cname  = pyrex_prefix + "filenames"
+fileinit_cname   = pyrex_prefix + "init_filenames"
+intern_tab_cname = pyrex_prefix + "intern_tab"
+kwds_cname       = pyrex_prefix + "kwds"
+lineno_cname     = pyrex_prefix + "lineno"
+clineno_cname    = pyrex_prefix + "clineno"
+cfilenm_cname    = pyrex_prefix + "cfilenm"
+module_cname     = pyrex_prefix + "m"
+moddoc_cname     = pyrex_prefix + "mdoc"
+methtable_cname  = pyrex_prefix + "methods"
+retval_cname     = pyrex_prefix + "r"
+reqd_kwds_cname  = pyrex_prefix + "reqd_kwds"
+self_cname       = pyrex_prefix + "self"
+stringtab_cname  = pyrex_prefix + "string_tab"
+vtabslot_cname   = pyrex_prefix + "vtab"
+c_api_tab_cname  = pyrex_prefix + "c_api_tab"
+gilstate_cname   = pyrex_prefix + "state"
+skip_dispatch_cname = pyrex_prefix + "skip_dispatch"
+empty_tuple      = pyrex_prefix + "empty_tuple"
+cleanup_cname    = pyrex_prefix + "module_cleanup"
+optional_args_cname = pyrex_prefix + "optional_args"
+no_opt_args      = pyrex_prefix + "no_opt_args"
+
+line_c_macro = "__LINE__"
+
+file_c_macro = "__FILE__"
+
+extern_c_macro  = pyrex_prefix.upper() + "EXTERN_C"
+
+exc_type_name   = pyrex_prefix + "exc_type"
+exc_value_name  = pyrex_prefix + "exc_value"
+exc_tb_name     = pyrex_prefix + "exc_tb"
+exc_lineno_name = pyrex_prefix + "exc_lineno"
+
+exc_vars = (exc_type_name, exc_value_name, exc_tb_name)
+
+api_name        = pyrex_prefix + "capi__"
+
+h_guard_prefix   = "__PYX_HAVE__"
+api_guard_prefix = "__PYX_HAVE_API__"
+api_func_guard   = "__PYX_HAVE_API_FUNC_"
+
+def py_version_hex(major, minor=0, micro=0, release_level=0, release_serial=0):
+    return (major << 24) | (minor << 16) | (micro << 8) | (release_level << 4) | (release_serial)
--- a/Cython/Compiler/Nodes.py
+++ b/Cython/Compiler/Nodes.py
--- a/Cython/Compiler/Options.py
+++ b/Cython/Compiler/Options.py
+#
+#  Pyrex - Compilation-wide options
+#
+
+intern_names = 1    #  Intern global variable and attribute names
+cache_builtins = 1  #  Perform lookups on builtin names only once
+
+embed_pos_in_docstring = 0
+gcc_branch_hints = 1
+
+pre_import = None
+docstrings = True
+
+# This is a SAGE-specific option that will 
+# cause Cython to incref local variables before
+# performing a binary operation on them, for 
+# safe detection of inplace operators. 
+incref_local_binop = 0
+
+# Decref global variables in this module on exit for garbage collection. 
+# 0: None, 1+: interned objects, 2+: cdef globals, 3+: types objects
+# Mostly for reducing noise for Valgrind, only executes at process exit
+# (when all memory will be reclaimed anyways). 
+generate_cleanup_code = 0
+
+annotate = 0
+
+# This will convert statements of the form "for i in range(...)" 
+# to "for i from ..." when i is a cdef'd integer type, and the direction
+# (i.e. sign of step) can be determined. 
+# WARNING: This may change the symantics if the range causes assignment to 
+# i to overflow. Specifically, if this option is set, an error will be
+# raised before the loop is entered, wheras without this option the loop
+# will execute util a overflowing value is encountered. 
+convert_range = 0
+
+# Enable this to allow one to write your_module.foo = ... to overwrite the 
+# definition if the cpdef function foo, at the cost of an extra dictionary 
+# lookup on every call. 
+# If this is 0 it simply creates a wrapper. 
+lookup_module_cpdef = 0
+
+# This will set local variables to None rather than NULL which may cause 
+# surpress what would be an UnboundLocalError in pure Python but eliminates 
+# checking for NULL on every use, and can decref rather than xdecref at the end. 
+# WARNING: This is a work in progress, may currently segfault.
+init_local_none = 1
+
+# Optimize no argument and one argument methods by using the METH_O and METH_NOARGS
+# calling conventions. These are faster calling conventions, but disallow the use of 
+# keywords (which, admittedly, are of little use in these cases). 
+optimize_simple_methods = 1
+
+# Append the c file and line number to the traceback for exceptions. 
+c_line_in_traceback = 0
--- a/Cython/Compiler/Parsing.py
+++ b/Cython/Compiler/Parsing.py
--- a/Cython/Compiler/PyrexTypes.py
+++ b/Cython/Compiler/PyrexTypes.py
--- a/Cython/Compiler/Scanning.py
+++ b/Cython/Compiler/Scanning.py
--- a/Cython/Compiler/Symtab.py
+++ b/Cython/Compiler/Symtab.py
--- a/Cython/Compiler/Transform.py
+++ b/Cython/Compiler/Transform.py
--- a/Cython/Compiler/TypeSlots.py
+++ b/Cython/Compiler/TypeSlots.py
--- a/Cython/Compiler/Version.py
+++ b/Cython/Compiler/Version.py
+version = '0.9.6.13'
--- a/Cython/Debugging.py
+++ b/Cython/Debugging.py
+###############################################
+#
+#   Odds and ends for debugging
+#
+###############################################
+
+def print_call_chain(*args):
+    import sys
+    print(" ".join(map(str, args)))
+    f = sys._getframe(1)
+    while f:
+        name = f.f_code.co_name
+        s = f.f_locals.get('self', None)
+        if s:
+            c = getattr(s, "__class__", None)
+            if c:
+                name = "%s.%s" % (c.__name__, name)
+        print("Called from: %s %s" % (name, f.f_lineno))
+        f = f.f_back
+    print("-" * 70)
--- a/Cython/Distutils/__init__.py
+++ b/Cython/Distutils/__init__.py
+# July 2002, Graham Fawcett
+#
+# this hack was inspired by the way Thomas Heller got py2exe
+# to appear as a distutil command
+#
+# we replace distutils.command.build_ext with our own version
+# and keep the old one under the module name _build_ext,
+# so that *our* build_ext can make use of it.
+
+from build_ext import build_ext
+# from extension import Extension
--- a/Cython/Distutils/build_ext.py
+++ b/Cython/Distutils/build_ext.py
--- a/Cython/Distutils/extension.py
+++ b/Cython/Distutils/extension.py
--- a/Cython/Mac/DarwinSystem.py
+++ b/Cython/Mac/DarwinSystem.py
--- a/Cython/Mac/MacSystem.py
+++ b/Cython/Mac/MacSystem.py
--- a/Cython/Mac/MacUtils.py
+++ b/Cython/Mac/MacUtils.py
+#
+#   Pyrex -- Misc Mac-specific things
+#
+
+import os, MacOS, macfs
+
+def open_new_file(path):
+    # On the Mac, try to preserve Finder position
+    # of previously existing file.
+    fsspec = macfs.FSSpec(path)
+    try:
+        old_finfo = fsspec.GetFInfo()
+    except MacOS.Error, e:
+        #print "MacUtils.open_new_file:", e ###
+        old_finfo = None
+    try:
+        os.unlink(path)
+    except OSError:
+        pass
+    file = open(path, "w")
+    new_finfo = fsspec.GetFInfo()
+    if old_finfo:
+        #print "MacUtils.open_new_file:", path ###
+        #print "...old file info =", old_finfo.Creator, old_finfo.Type, old_finfo.Location ###
+        #print "...new file info =", new_finfo.Creator, new_finfo.Type, new_finfo.Location ###
+        new_finfo.Location = old_finfo.Location
+        new_finfo.Flags = old_finfo.Flags
+    # Make darn sure the type and creator are right. There seems
+    # to be a bug in MacPython 2.2 that screws them up sometimes.
+    new_finfo.Creator = "R*ch"
+    new_finfo.Type = "TEXT"
+    fsspec.SetFInfo(new_finfo)
+    return file
+
--- a/Cython/Mac/Makefile
+++ b/Cython/Mac/Makefile
+# Makefile for Darwin
+
+# Change this to your Python source location
+PYTHON := /Local/Build/Pythonic/python/2.3
+
+INCLUDE := -I$(PYTHON) -I$(PYTHON)/Include -I$(PYTHON)/Mac/Include
+
+CCOPTS := -fno-strict-aliasing -Wno-long-double -no-cpp-precomp \
+	-mno-fused-madd -fno-common -dynamic
+
+LDOPTS := -Wl,-F.,-w -bundle -framework Python -framework Carbon
+
+all:	_File.so
+
+_File.o:	_Filemodule_patched.c
+	gcc -c $(INCLUDE) $(OPTS) $< -o $@
+
+_File.so:	_File.o
+	gcc $(LDOPTS) $< -o $@
--- a/Cython/Mac/TS_Misc_Suite.py
+++ b/Cython/Mac/TS_Misc_Suite.py
--- a/Cython/Mac/_Filemodule_patched.c
+++ b/Cython/Mac/_Filemodule_patched.c
--- a/Cython/Plex/Actions.py
+++ b/Cython/Plex/Actions.py
--- a/Cython/Plex/DFA.py
+++ b/Cython/Plex/DFA.py
--- a/Cython/Plex/Errors.py
+++ b/Cython/Plex/Errors.py
--- a/Cython/Plex/Lexicons.py
+++ b/Cython/Plex/Lexicons.py
--- a/Cython/Plex/Machines.py
+++ b/Cython/Plex/Machines.py
--- a/Cython/Plex/Regexps.py
+++ b/Cython/Plex/Regexps.py
--- a/Cython/Plex/Scanners.py
+++ b/Cython/Plex/Scanners.py
--- a/Cython/Plex/Timing.py
+++ b/Cython/Plex/Timing.py
--- a/Cython/Plex/Traditional.py
+++ b/Cython/Plex/Traditional.py
--- a/Cython/Plex/Transitions.py
+++ b/Cython/Plex/Transitions.py
--- a/Cython/Plex/__init__.py
+++ b/Cython/Plex/__init__.py
--- a/Cython/Plex/test_tm.py
+++ b/Cython/Plex/test_tm.py
--- a/Cython/Unix/LinuxSystem.py
+++ b/Cython/Unix/LinuxSystem.py
--- a/Cython/Utils.py
+++ b/Cython/Utils.py